Repository: google/exposure-notifications-server
Pull request #1483 (2021-03-22 23:38:12)

Use an intermediate proxy for backups
Unfortunately this is required to measure forward progress.

**Release Note**

<!-- Enter your extended release note in the below block. If the PR requires
additional action from users switching to the new release, include the string
"action required". If no release note is required, write "NONE". -->

```release-note
Introduce a new service: `backup`. Be sure to taint the null_resource builder during the Terraform apply to get the new service version.
```

sethvargo: @mikehelmick ready for another review

Patch:
diff --git a/builders/build.yaml b/builders/build.yaml
index a11dbf0..ed430de 100644
--- a/builders/build.yaml
+++ b/builders/build.yaml
@@ -130,6 +130,45 @@ steps:
   waitFor:
   - 'push-admin-console'
 
+#
+# backup
+#
+- id: 'dockerize-backup'
+  name: 'docker:19'
+  args:
+  - 'build'
+  - '--file=builders/service.dockerfile'
+  - '--tag=gcr.io/${PROJECT_ID}/${_REPO}/backup:${_TAG}'
+  - '--build-arg=SERVICE=backup'
+  - '.'
+  waitFor:
+  - 'build'
+
+- id: 'push-backup'
+  name: 'docker:19'
+  args:
+  - 'push'
+  - 'gcr.io/${PROJECT_ID}/${_REPO}/backup:${_TAG}'
+  waitFor:
+  - 'dockerize-backup'
+
+- id: 'attest-backup'
+  name: 'gcr.io/google.com/cloudsdktool/cloud-sdk:324.0.0'
+  args:
+  - 'bash'
+  - '-eEuo'
+  - 'pipefail'
+  - '-c'
+  - |-
+    ARTIFACT_URL=$(docker inspect gcr.io/${PROJECT_ID}/${_REPO}/backup:${_TAG} --format='{{index .RepoDigests 0}}')
+    gcloud beta container binauthz attestations sign-and-create \
+      --project "${PROJECT_ID}" \
+      --artifact-url "$${ARTIFACT_URL}" \
+      --attestor "${_BINAUTHZ_ATTESTOR}" \
+      --keyversion "${_BINAUTHZ_KEY_VERSION}"
+  waitFor:
+  - 'push-backup'
+
 #
 # cleanup-export
 #
diff --git a/builders/deploy.yaml b/builders/deploy.yaml
index 4e9fdcb..0ef4ccc 100644
--- a/builders/deploy.yaml
+++ b/builders/deploy.yaml
@@ -43,6 +43,27 @@ steps:
   waitFor:
   - '-'
 
+#
+# backup
+#
+- id: 'deploy-backup'
+  name: 'gcr.io/google.com/cloudsdktool/cloud-sdk:324.0.0-alpine'
+  args:
+  - 'bash'
+  - '-eEuo'
+  - 'pipefail'
+  - '-c'
+  - |-
+    gcloud run deploy "backup" \
+      --quiet \
+      --project "${PROJECT_ID}" \
+      --platform "managed" \
+      --region "${_REGION}" \
+      --image "gcr.io/${PROJECT_ID}/${_REPO}/backup:${_TAG}" \
+      --no-traffic
+  waitFor:
+  - '-'
+
 #
 # cleanup-exposure
 #
diff --git a/builders/promote.yaml b/builders/promote.yaml
index b3fe446..ea8840e 100644
--- a/builders/promote.yaml
+++ b/builders/promote.yaml
@@ -42,6 +42,26 @@ steps:
   waitFor:
   - '-'
 
+#
+# backup
+#
+- id: 'promote-backup'
+  name: 'gcr.io/google.com/cloudsdktool/cloud-sdk:324.0.0-alpine'
+  args:
+  - 'bash'
+  - '-eEuo'
+  - 'pipefail'
+  - '-c'
+  - |-
+    gcloud run services update-traffic "backup" \
+      --quiet \
+      --project "${PROJECT_ID}" \
+      --platform "managed" \
+      --region "${_REGION}" \
+      --to-revisions "${_REVISION}=${_PERCENTAGE}"
+  waitFor:
+  - '-'
+
 #
 # cleanup-exposure
 #
diff --git a/cmd/backup/main.go b/cmd/backup/main.go
new file mode 100644
index 0000000..723d999
--- /dev/null
+++ b/cmd/backup/main.go
@@ -0,0 +1,77 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// This package is the service that deletes old exposure keys; it is intended to be invoked over HTTP by Cloud Scheduler.
+package main
+
+import (
+	"context"
+	"fmt"
+	"os/signal"
+	"syscall"
+
+	"github.com/google/exposure-notifications-server/internal/backup"
+	"github.com/google/exposure-notifications-server/internal/buildinfo"
+	"github.com/google/exposure-notifications-server/internal/setup"
+	"github.com/google/exposure-notifications-server/pkg/logging"
+	"github.com/google/exposure-notifications-server/pkg/server"
+)
+
+func main() {
+	ctx, done := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM)
+
+	logger := logging.NewLoggerFromEnv().
+		With("build_id", buildinfo.BuildID).
+		With("build_tag", buildinfo.BuildTag)
+	ctx = logging.WithLogger(ctx, logger)
+
+	defer func() {
+		done()
+		if r := recover(); r != nil {
+			logger.Fatalw("application panic", "panic", r)
+		}
+	}()
+
+	err := realMain(ctx)
+	done()
+
+	if err != nil {
+		logger.Fatal(err)
+	}
+	logger.Info("successful shutdown")
+}
+
+func realMain(ctx context.Context) error {
+	logger := logging.FromContext(ctx)
+
+	var config backup.Config
+	env, err := setup.Setup(ctx, &config)
+	if err != nil {
+		return fmt.Errorf("setup.Setup: %w", err)
+	}
+	defer env.Close(ctx)
+
+	backupServer, err := backup.NewServer(&config, env)
+	if err != nil {
+		return fmt.Errorf("backup.NewServer: %w", err)
+	}
+
+	srv, err := server.New(config.Port)
+	if err != nil {
+		return fmt.Errorf("server.New: %w", err)
+	}
+	logger.Info("listening on: ", config.Port)
+
+	return srv.ServeHTTPHandler(ctx, backupServer.Routes(ctx))
+}
diff --git a/internal/backup/backup.go b/internal/backup/backup.go
new file mode 100644
index 0000000..7d2b4c0
--- /dev/null
+++ b/internal/backup/backup.go
@@ -0,0 +1,16 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Package backup implements periodic backups.
+package backup
diff --git a/internal/backup/config.go b/internal/backup/config.go
new file mode 100644
index 0000000..7b49c13
--- /dev/null
+++ b/internal/backup/config.go
@@ -0,0 +1,73 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package backup
+
+import (
+	"time"
+
+	"github.com/google/exposure-notifications-server/internal/setup"
+	"github.com/google/exposure-notifications-server/pkg/database"
+	"github.com/google/exposure-notifications-server/pkg/observability"
+	"github.com/google/exposure-notifications-server/pkg/secrets"
+)
+
+// Compile-time check to assert this config matches requirements.
+var (
+	_ setup.DatabaseConfigProvider              = (*Config)(nil)
+	_ setup.ObservabilityExporterConfigProvider = (*Config)(nil)
+	_ setup.SecretManagerConfigProvider         = (*Config)(nil)
+)
+
+// Config represents the configuration and associated environment variables for
+// the cleanup components.
+type Config struct {
+	Database              database.Config
+	ObservabilityExporter observability.Config
+	SecretManager         secrets.Config
+
+	Port string `env:"PORT, default=8080"`
+
+	// MinTTL is the minimum amount of time that must elapse between attempting
+	// backups. This is used to control whether the pull is actually attempted at
+	// the controller layer, independent of the data layer. In effect, it rate
+	// limits the number of requests.
+	MinTTL time.Duration `env:"BACKUP_MIN_PERIOD, default=4h"`
+
+	// Timeout is the maximum amount of time to wait for a backup operation to
+	// complete.
+	Timeout time.Duration `env:"BACKUP_TIMEOUT, default=10m"`
+
+	// Bucket is the name of the Cloud Storage bucket where backups should be
+	// stored.
+	Bucket string `env:"BACKUP_BUCKET, required"`
+
+	// DatabaseInstanceURL is the full self-link of the URL to the SQL instance.
+	DatabaseInstanceURL string `env:"BACKUP_DATABASE_INSTANCE_URL, required"`
+
+	// DatabaseName is the name of the database to backup.
+	DatabaseName string `env:"BACKUP_DATABASE_NAME, required"`
+}
+
+func (c *Config) DatabaseConfig() *database.Config {
+	return &c.Database
+}
+
+func (c *Config) ObservabilityExporterConfig() *observability.Config {
+	return &c.ObservabilityExporter
+}
+
+func (c *Config) SecretManagerConfig() *secrets.Config {
+	return &c.SecretManager
+}
diff --git a/internal/backup/handle_backup.go b/internal/backup/handle_backup.go
new file mode 100644
index 0000000..d087005
--- /dev/null
+++ b/internal/backup/handle_backup.go
@@ -0,0 +1,161 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package backup
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"path"
+
+	"github.com/google/exposure-notifications-server/pkg/database"
+	"github.com/google/exposure-notifications-server/pkg/logging"
+	"github.com/kelseyhightower/run"
+	"go.opencensus.io/stats"
+)
+
+const backupDatabaseLockID = "backup-database-lock" // TODO
+
+func (s *Server) handleBackup() http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		ctx := r.Context()
+
+		logger := logging.FromContext(ctx).Named("backup.HandleBackup")
+
+		req, err := s.buildBackupRequest(ctx)
+		if err != nil {
+			logger.Errorw("failed to build request", "error", err)
+			s.h.RenderJSON(w, http.StatusInternalServerError, err)
+			return
+		}
+
+		unlock, err := s.db.Lock(ctx, backupDatabaseLockID, s.config.MinTTL)
+		if err != nil {
+			if errors.Is(err, database.ErrAlreadyLocked) {
+				logger.Debugw("skipping (already locked)")
+				s.h.RenderJSON(w, http.StatusOK, fmt.Errorf("too early"))
+				return
+			}
+			logger.Errorw("failed to obtain lock", "error", err)
+			s.h.RenderJSON(w, http.StatusInternalServerError, err)
+			return
+		}
+		releaseLock := func() {
+			// Note that we explicitly don't release the lock because it allows us to
+			// enforce the minimum TTL. By setting the maximum TTL to the minimum TTL
+			// and not releasing the lock, it means the next request will fail to
+			// acquire the lock (because it is held). However, after the TTL elapses,
+			// the lock will have been expired and the run can successfully continue.
+			if err := unlock(); err != nil {
+				logger.Errorw("failed to unlock", "error", err)
+			}
+		}
+
+		if err := s.executeBackup(req); err != nil {
+			defer releaseLock()
+			logger.Errorw("failed to execute backup", "error", err)
+			s.h.RenderJSON(w, http.StatusInternalServerError, err)
+			return
+		}
+
+		stats.Record(ctx, mSuccess.M(1))
+		s.h.RenderJSON(w, http.StatusOK, nil)
+	})
+}
+
+func (s *Server) buildBackupRequest(ctx context.Context) (*http.Request, error) {
+	u, err := url.Parse(s.config.DatabaseInstanceURL)
+	if err != nil {
+		return nil, fmt.Errorf("failed to parse database instance url: %w", err)
+	}
+	u.Path = path.Join(u.Path, "export")
+
+	token, err := s.authorizationToken(ctx)
+	if err != nil {
+		return nil, fmt.Errorf("failed to get authorization token: %w", err)
+	}
+
+	var b bytes.Buffer
+	if err := json.NewEncoder(&b).Encode(&backupRequest{
+		ExportContext: &exportContext{
+			FileType:  "SQL",
+			URI:       fmt.Sprintf("gs://%s/database/%s", s.config.Bucket, s.config.DatabaseName),
+			Databases: []string{s.config.DatabaseName},
+
+			// Specifically disable offloading because we want this request to run
+			// in-band so we can verify the return status.
+			Offload: false,
+		},
+	}); err != nil {
+		return nil, fmt.Errorf("failed to create body: %w", err)
+	}
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, u.String(), &b)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create request: %w", err)
+	}
+	req.Header.Set("Authorization", "Bearer "+token)
+
+	return req, nil
+}
+
+func (s *Server) authorizationToken(ctx context.Context) (string, error) {
+	if v := s.overrideAuthToken; v != "" {
+		return v, nil
+	}
+
+	token, err := run.Token([]string{"https://www.googleapis.com/auth/cloud-platform"})
+	if err != nil {
+		return "", fmt.Errorf("failed to get token: %w", err)
+	}
+	return token.AccessToken, nil
+}
+
+// executeBackup calls the backup API. This is a *blocking* operation that can
+// take O(minutes) in some cases.
+func (s *Server) executeBackup(req *http.Request) error {
+	client := &http.Client{
+		Timeout: s.config.Timeout,
+	}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("failed to execute request: %w", err)
+	}
+	defer resp.Body.Close()
+
+	if got, want := resp.StatusCode, http.StatusOK; got != want {
+		b, _ := io.ReadAll(resp.Body)
+		return fmt.Errorf("unsuccessful response from backup (got %d, want %d): %s", got, want, b)
+	}
+
+	return nil
+}
+
+type backupRequest struct {
+	ExportContext *exportContext `json:"exportContext"`
+}
+
+type exportContext struct {
+	FileType  string   `json:"fileType"`
+	URI       string   `json:"uri"`
+	Databases []string `json:"databases"`
+	Offload   bool     `json:"offload"`
+}
diff --git a/internal/backup/metrics.go b/internal/backup/metrics.go
new file mode 100644
index 0000000..ae67468
--- /dev/null
+++ b/internal/backup/metrics.go
@@ -0,0 +1,37 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package backup
+
+import (
+	"github.com/google/exposure-notifications-server/internal/metrics"
+	"github.com/google/exposure-notifications-server/pkg/observability"
+	"go.opencensus.io/stats"
+	"go.opencensus.io/stats/view"
+)
+
+const metricPrefix = metrics.MetricRoot + "backup"
+
+var mSuccess = stats.Int64(metricPrefix+"/success", "successful execution", stats.UnitDimensionless)
+
+func init() {
+	observability.CollectViews([]*view.View{
+		{
+			Name:        metricPrefix + "/success",
+			Description: "Number of successes",
+			Measure:     mSuccess,
+			Aggregation: view.Count(),
+		},
+	}...)
+}
diff --git a/internal/backup/server.go b/internal/backup/server.go
new file mode 100644
index 0000000..c6f3d3a
--- /dev/null
+++ b/internal/backup/server.go
@@ -0,0 +1,70 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package backup
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/google/exposure-notifications-server/internal/middleware"
+	"github.com/google/exposure-notifications-server/internal/serverenv"
+	"github.com/google/exposure-notifications-server/pkg/database"
+	"github.com/google/exposure-notifications-server/pkg/logging"
+	"github.com/google/exposure-notifications-server/pkg/render"
+	"github.com/google/exposure-notifications-server/pkg/server"
+	"github.com/gorilla/mux"
+)
+
+type Server struct {
+	config *Config
+	env    *serverenv.ServerEnv
+	db     *database.DB
+	h      *render.Renderer
+
+	// overrideAuthToken is for testing to bypass API calls to get authentication
+	// information.
+	overrideAuthToken string
+}
+
+func NewServer(config *Config, env *serverenv.ServerEnv) (*Server, error) {
+	if env.Database() == nil {
+		return nil, fmt.Errorf("missing database in server environment")
+	}
+
+	db := env.Database()
+
+	return &Server{
+		config: config,
+		env:    env,
+		db:     db,
+		h:      render.NewRenderer(),
+	}, nil
+}
+
+// Routes defines and returns the routes for this server.
+func (s *Server) Routes(ctx context.Context) *mux.Router {
+	logger := logging.FromContext(ctx).Named("backup")
+
+	r := mux.NewRouter()
+	r.Use(middleware.Recovery())
+	r.Use(middleware.PopulateRequestID())
+	r.Use(middleware.PopulateObservability())
+	r.Use(middleware.PopulateLogger(logger))
+
+	r.Handle("/health", server.HandleHealthz(s.env.Database()))
+	r.Handle("/", s.handleBackup())
+
+	return r
+}
diff --git a/terraform/alerting/alerts.tf b/terraform/alerting/alerts.tf
index aa0f560..91553ab 100644
--- a/terraform/alerting/alerts.tf
+++ b/terraform/alerting/alerts.tf
@@ -19,6 +19,9 @@ locals {
 
   forward_progress_indicators = merge(
     {
+      // backup runs every 4h, alert after 2 failures
+      "backup" = { metric = "backup/success", window = "485m" },
+
       // cleanup-export runs every 4h, alert after 2 failures
       "cleanup-export" = { metric = "cleanup/export/success", window = "485m" },
 
diff --git a/terraform/database.tf b/terraform/database.tf
index ec8b1b5..3a0b6f8 100644
--- a/terraform/database.tf
+++ b/terraform/database.tf
@@ -252,59 +252,6 @@ resource "google_storage_bucket_iam_member" "instance-objectAdmin" {
   member = "serviceAccount:${google_sql_database_instance.db-inst.service_account_email_address}"
 }
 
-# Create a service account that can initiate backups.
-resource "google_service_account" "database-backuper" {
-  project      = data.google_project.project.project_id
-  account_id   = "en-database-backuper"
-  display_name = "Key Server database backuper"
-}
-
-# Give the service account the ability to initiate backups - unfortunately this
-# has to be granted at the project level and isn't scoped to a single database
-# instance.
-resource "google_project_iam_member" "database-backuper-cloudsql-viewer" {
-  project = var.project
-  role    = "roles/cloudsql.viewer"
-  member  = "serviceAccount:${google_service_account.database-backuper.email}"
-}
-
-# Create a Cloud Scheduler job that generates a backup every interval.
-resource "google_cloud_scheduler_job" "backup-database-worker" {
-  name             = "backup-database-worker"
-  region           = var.cloudscheduler_location
-  schedule         = var.database_backup_schedule
-  time_zone        = "America/Los_Angeles"
-  attempt_deadline = "1800s"
-
-  retry_config {
-    retry_count = 3
-  }
-
-  http_target {
-    http_method = "POST"
-    uri         = "${google_sql_database_instance.db-inst.self_link}/export"
-
-    body = base64encode(jsonencode({
-      exportContext = {
-        fileType  = "SQL"
-        uri       = "gs://${google_storage_bucket.backups.name}/database/${google_sql_database.db.name}",
-        databases = [google_sql_database.db.name]
-        offload   = false,
-      }
-    }))
-
-    oauth_token {
-      service_account_email = google_service_account.database-backuper.email
-    }
-  }
-
-  depends_on = [
-    google_app_engine_application.app,
-    google_project_iam_member.database-backuper-cloudsql-viewer,
-    google_project_service.services["cloudscheduler.googleapis.com"],
-  ]
-}
-
 output "db_conn" {
   value = google_sql_database_instance.db-inst.connection_name
 }
diff --git a/terraform/service_admin_console.tf b/terraform/service_admin_console.tf
index a52e73f..05c0ebf 100644
--- a/terraform/service_admin_console.tf
+++ b/terraform/service_admin_console.tf
@@ -95,8 +95,6 @@ resource "google_cloud_run_service" "admin-console" {
           for_each = merge(
             local.common_cloudrun_env_vars,
 
-            { "TEMPLATE_DIR" : "/templates" },
-
             // This MUST come last to allow overrides!
             lookup(var.service_environment, "admin-console", {}),
           )
diff --git a/terraform/service_backup.tf b/terraform/service_backup.tf
new file mode 100644
index 0000000..d0532cb
--- /dev/null
+++ b/terraform/service_backup.tf
@@ -0,0 +1,196 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+resource "google_service_account" "backup" {
+  project      = data.google_project.project.project_id
+  account_id   = "en-backup-sa"
+  display_name = "Exposure Notification Backup"
+}
+
+resource "google_service_account_iam_member" "cloudbuild-deploy-backup" {
+  service_account_id = google_service_account.backup.id
+  role               = "roles/iam.serviceAccountUser"
+  member             = "serviceAccount:${data.google_project.project.number}@cloudbuild.gserviceaccount.com"
+
+  depends_on = [
+    google_project_service.services["cloudbuild.googleapis.com"],
+  ]
+}
+
+resource "google_secret_manager_secret_iam_member" "backup-db" {
+  for_each = toset([
+    "sslcert",
+    "sslkey",
+    "sslrootcert",
+    "password",
+  ])
+
+  secret_id = google_secret_manager_secret.db-secret[each.key].id
+  role      = "roles/secretmanager.secretAccessor"
+  member    = "serviceAccount:${google_service_account.backup.email}"
+}
+
+resource "google_project_iam_member" "backup-observability" {
+  for_each = toset([
+    "roles/cloudtrace.agent",
+    "roles/logging.logWriter",
+    "roles/monitoring.metricWriter",
+    "roles/stackdriver.resourceMetadata.writer",
+  ])
+
+  project = var.project
+  role    = each.key
+  member  = "serviceAccount:${google_service_account.backup.email}"
+}
+
+# Give the service account the ability to initiate backups - unfortunately this
+# has to be granted at the project level and isn't scoped to a single database
+# instance.
+resource "google_project_iam_member" "database-backuper-cloudsql-viewer" {
+  project = var.project
+  role    = "roles/cloudsql.viewer"
+  member  = "serviceAccount:${google_service_account.backup.email}"
+}
+
+resource "google_cloud_run_service" "backup" {
+  name     = "backup"
+  location = var.cloudrun_location
+
+  autogenerate_revision_name = true
+
+  metadata {
+    annotations = merge(
+      local.default_service_annotations,
+      var.default_service_annotations_overrides,
+      lookup(var.service_annotations, "backup", {}),
+    )
+  }
+
+  template {
+    spec {
+      service_account_name = google_service_account.backup.email
+      timeout_seconds      = 900
+
+      containers {
+        image = "gcr.io/${data.google_project.project.project_id}/github.com/google/exposure-notifications-server/backup:initial"
+
+        resources {
+          limits = {
+            cpu    = "1"
+            memory = "512Mi"
+          }
+        }
+
+        dynamic "env" {
+          for_each = merge(
+            local.common_cloudrun_env_vars,
+
+            {
+              BACKUP_BUCKET                = google_storage_bucket.backups.name,
+              BACKUP_DATABASE_INSTANCE_URL = google_sql_database_instance.db-inst.self_link,
+              BACKUP_DATABASE_NAME         = google_sql_database.db.name,
+            },
+
+            // This MUST come last to allow overrides!
+            lookup(var.service_environment, "backup", {}),
+          )
+
+          content {
+            name  = env.key
+            value = env.value
+          }
+        }
+      }
+    }
+
+    metadata {
+      annotations = merge(
+        local.default_revision_annotations,
+        var.default_revision_annotations_overrides,
+        lookup(var.revision_annotations, "backup", {}),
+      )
+    }
+  }
+
+  depends_on = [
+    google_project_service.services["run.googleapis.com"],
+    google_secret_manager_secret_iam_member.backup-db,
+    null_resource.build,
+    null_resource.migrate,
+  ]
+
+  lifecycle {
+    ignore_changes = [
+      metadata[0].annotations["client.knative.dev/user-image"],
+      metadata[0].annotations["run.googleapis.com/client-name"],
+      metadata[0].annotations["run.googleapis.com/client-version"],
+      metadata[0].annotations["run.googleapis.com/ingress-status"],
+      metadata[0].annotations["serving.knative.dev/creator"],
+      metadata[0].annotations["serving.knative.dev/lastModifier"],
+      metadata[0].labels["cloud.googleapis.com/location"],
+      template[0].metadata[0].annotations["client.knative.dev/user-image"],
+      template[0].metadata[0].annotations["run.googleapis.com/client-name"],
+      template[0].metadata[0].annotations["run.googleapis.com/client-version"],
+      template[0].metadata[0].annotations["serving.knative.dev/creator"],
+      template[0].metadata[0].annotations["serving.knative.dev/lastModifier"],
+      template[0].spec[0].containers[0].image,
+    ]
+  }
+}
+
+
+#
+# Create scheduler job to invoke the service on a fixed interval.
+#
+
+resource "google_service_account" "backup-invoker" {
+  project      = data.google_project.project.project_id
+  account_id   = "en-backup-invoker-sa"
+  display_name = "Exposure Notification Backup Invoker"
+}
+
+resource "google_cloud_run_service_iam_member" "backup-invoker" {
+  project  = google_cloud_run_service.backup.project
+  location = google_cloud_run_service.backup.location
+  service  = google_cloud_run_service.backup.name
+  role     = "roles/run.invoker"
+  member   = "serviceAccount:${google_service_account.backup-invoker.email}"
+}
+
+resource "google_cloud_scheduler_job" "backup-worker" {
+  name             = "backup-worker"
+  region           = var.cloudscheduler_location
+  schedule         = "0 */4 * * *"
+  time_zone        = "America/Los_Angeles"
+  attempt_deadline = "${google_cloud_run_service.backup.template[0].spec[0].timeout_seconds + 60}s"
+
+  retry_config {
+    retry_count = 3
+  }
+
+  http_target {
+    http_method = "POST"
+    uri         = "${google_cloud_run_service.backup.status.0.url}/"
+    oidc_token {
+      audience              = google_cloud_run_service.backup.status.0.url
+      service_account_email = google_service_account.backup-invoker.email
+    }
+  }
+
+  depends_on = [
+    google_app_engine_application.app,
+    google_cloud_run_service_iam_member.backup-invoker,
+    google_project_service.services["cloudscheduler.googleapis.com"],
+  ]
+}


Test patch:
diff --git a/internal/backup/backup_test.go b/internal/backup/backup_test.go
new file mode 100644
index 0000000..a99f24c
--- /dev/null
+++ b/internal/backup/backup_test.go
@@ -0,0 +1,29 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package backup
+
+import (
+	"testing"
+
+	"github.com/google/exposure-notifications-server/pkg/database"
+)
+
+var testDatabaseInstance *database.TestInstance
+
+func TestMain(m *testing.M) {
+	testDatabaseInstance = database.MustTestInstance()
+	defer testDatabaseInstance.MustClose()
+	m.Run()
+}
diff --git a/internal/backup/handle_backup_test.go b/internal/backup/handle_backup_test.go
new file mode 100644
index 0000000..9f6a024
--- /dev/null
+++ b/internal/backup/handle_backup_test.go
@@ -0,0 +1,200 @@
+// Copyright 2021 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package backup
+
+import (
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/google/exposure-notifications-server/internal/project"
+	"github.com/google/exposure-notifications-server/internal/serverenv"
+)
+
+func TestServer_HandleBackup(t *testing.T) {
+	t.Parallel()
+
+	ctx := project.TestContext(t)
+
+	t.Run("bad_database", func(t *testing.T) {
+		t.Parallel()
+
+		testDB, _ := testDatabaseInstance.NewDatabase(t)
+		env := serverenv.New(ctx, serverenv.WithDatabase(testDB))
+
+		cfg := &Config{
+			DatabaseInstanceURL: "https://example.com",
+		}
+		s, err := NewServer(cfg, env)
+		if err != nil {
+			t.Fatal(err)
+		}
+		s.overrideAuthToken = "token"
+		handler := s.handleBackup()
+
+		r, err := http.NewRequestWithContext(ctx, http.MethodGet, "/", nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+		w := httptest.NewRecorder()
+		handler.ServeHTTP(w, r)
+
+		if got, want := w.Code, http.StatusInternalServerError; got != want {
+			t.Errorf("expected %d to be %d: %s", got, want, w.Body)
+		}
+	})
+
+	t.Run("bad_endpoint", func(t *testing.T) {
+		t.Parallel()
+
+		testDB, _ := testDatabaseInstance.NewDatabase(t)
+		env := serverenv.New(ctx, serverenv.WithDatabase(testDB))
+
+		cfg := &Config{
+			DatabaseInstanceURL: "\x7f",
+		}
+		s, err := NewServer(cfg, env)
+		if err != nil {
+			t.Fatal(err)
+		}
+		s.overrideAuthToken = "token"
+		handler := s.handleBackup()
+
+		r, err := http.NewRequestWithContext(ctx, http.MethodGet, "/", nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+		w := httptest.NewRecorder()
+		handler.ServeHTTP(w, r)
+
+		if got, want := w.Code, http.StatusInternalServerError; got != want {
+			t.Errorf("expected %d to be %d: %s", got, want, w.Body)
+		}
+		if got, want := w.Body.String(), "failed to parse database instance url"; !strings.Contains(got, want) {
+			t.Errorf("expected body to contain %q: %s", want, got)
+		}
+	})
+
+	t.Run("bad_url", func(t *testing.T) {
+		t.Parallel()
+
+		testDB, _ := testDatabaseInstance.NewDatabase(t)
+		env := serverenv.New(ctx, serverenv.WithDatabase(testDB))
+
+		cfg := &Config{
+			DatabaseInstanceURL: "https://not-a-real.web.site.no",
+		}
+		s, err := NewServer(cfg, env)
+		if err != nil {
+			t.Fatal(err)
+		}
+		s.overrideAuthToken = "token"
+		handler := s.handleBackup()
+
+		r, err := http.NewRequestWithContext(ctx, http.MethodGet, "/", nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+		w := httptest.NewRecorder()
+		handler.ServeHTTP(w, r)
+
+		if got, want := w.Code, http.StatusInternalServerError; got != want {
+			t.Errorf("expected %d to be %d: %s", got, want, w.Body)
+		}
+		if got, want := w.Body.String(), "failed to execute request"; !strings.Contains(got, want) {
+			t.Errorf("expected body to contain %q: %s", want, got)
+		}
+	})
+
+	t.Run("success", func(t *testing.T) {
+		t.Parallel()
+
+		testDB, _ := testDatabaseInstance.NewDatabase(t)
+		env := serverenv.New(ctx, serverenv.WithDatabase(testDB))
+
+		ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.WriteHeader(200)
+		}))
+		defer ts.Close()
+
+		cfg := &Config{
+			MinTTL:              15 * time.Second,
+			Bucket:              "bucket",
+			DatabaseInstanceURL: ts.URL + "/instance",
+			DatabaseName:        "name",
+		}
+
+		s, err := NewServer(cfg, env)
+		if err != nil {
+			t.Fatal(err)
+		}
+		s.overrideAuthToken = "token"
+		handler := s.handleBackup()
+
+		r, err := http.NewRequestWithContext(ctx, http.MethodGet, "/", nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+		w := httptest.NewRecorder()
+		handler.ServeHTTP(w, r)
+
+		if got, want := w.Code, http.StatusOK; got != want {
+			t.Errorf("expected %d to be %d: %s", got, want, w.Body)
+		}
+	})
+
+	t.Run("bad_upstream_response", func(t *testing.T) {
+		t.Parallel()
+
+		testDB, _ := testDatabaseInstance.NewDatabase(t)
+		env := serverenv.New(ctx, serverenv.WithDatabase(testDB))
+
+		ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			w.WriteHeader(500)
+			fmt.Fprint(w, "badness")
+		}))
+		defer ts.Close()
+
+		cfg := &Config{
+			Bucket:              "bucket",
+			DatabaseInstanceURL: ts.URL + "/instance",
+			DatabaseName:        "name",
+		}
+
+		s, err := NewServer(cfg, env)
+		if err != nil {
+			t.Fatal(err)
+		}
+		s.overrideAuthToken = "token"
+		handler := s.handleBackup()
+
+		r, err := http.NewRequestWithContext(ctx, http.MethodGet, "/", nil)
+		if err != nil {
+			t.Fatal(err)
+		}
+		w := httptest.NewRecorder()
+		handler.ServeHTTP(w, r)
+
+		if got, want := w.Code, http.StatusInternalServerError; got != want {
+			t.Errorf("expected %d to be %d: %s", got, want, w.Body)
+		}
+		if got, want := w.Body.String(), "unsuccessful response from backup"; !strings.Contains(got, want) {
+			t.Errorf("expected body to contain %q: %s", want, got)
+		}
+	})
+}