Repository: google/exposure-notifications-server
Pull request #1224 (2020-12-10 08:48:55)

Use proper unit and avoid converting between units.
It looks like the 'm' in 'ms' is a PREFIX_SYMBOL of 's' and we can use
's' as unit directly.

Reference: https://cloud.google.com/monitoring/mql/reference#units-of-measure

google-oss-robot: [APPROVALNOTIFIER] This PR is **NOT APPROVED**

This pull-request has been approved by: *<a href="https://github.com/google/exposure-notifications-server/pull/1224#" title="Author self-approved">yegle</a>*
To complete the [pull request process](https://git.k8s.io/community/contributors/guide/owners.md#the-code-review-process), please assign **sethvargo** after the PR has been reviewed.
You can assign the PR to them by writing `/assign @sethvargo` in a comment when ready.

The full list of commands accepted by this bot can be found [here](https://go.k8s.io/bot-commands?repo=google%2Fexposure-notifications-server).

<details open>
Needs approval from an approver in each of these files:

- **[terraform/OWNERS](https://github.com/google/exposure-notifications-server/blob/main/terraform/OWNERS)**

Approvers can indicate their approval by writing `/approve` in a comment
Approvers can cancel approval by writing `/approve cancel` in a comment
</details>
<!-- META={"approvers":["sethvargo"]} -->
google-oss-robot: @yegle: The following test **failed**, say `/retest` to rerun all failed tests:

Test name | Commit | Details | Rerun command
--- | --- | --- | ---
pull-en-server-release-unit | 58a683458f0537046e9ff080203d3119297d15fa | [link](https://oss-prow.knative.dev/view/gs/oss-prow/pr-logs/pull/google_exposure-notifications-server/1224/pull-en-server-release-unit/1336958596172222464) | `/test pull-en-server-release-unit`

<details>

Instructions for interacting with me using PR comments are available [here](https://git.k8s.io/community/contributors/guide/pull-requests.md).  If you have questions or suggestions related to my behavior, please file an issue against the [kubernetes/test-infra](https://github.com/kubernetes/test-infra/issues/new?title=Prow%20issue:) repository. I understand the commands that are listed [here](https://go.k8s.io/bot-commands).
</details>
<!-- test report -->

Patch:
diff --git a/docs/playbooks/alerts/LatencyTooHigh.md b/docs/playbooks/alerts/LatencyTooHigh.md
new file mode 100644
index 0000000..488522d
--- /dev/null
+++ b/docs/playbooks/alerts/LatencyTooHigh.md
@@ -0,0 +1,12 @@
+# LatencyTooHigh
+
+This alert is firing when the response latency is too high for one or
+more services.
+
+Check whether the latency is expected. Not all latency increase
+have user impact and some services (e.g. export, cleanup) is expected to
+have higher latency. If that's the case, consider adjust the alerting
+threshold in [alerts.tf](../../../terraform/alerting/alerts.tf).
+
+Reach out to internal chat if it cannot be explained by alerting
+threshold.
diff --git a/go.mod b/go.mod
index 643da06..d7a1be7 100644
--- a/go.mod
+++ b/go.mod
@@ -53,6 +53,7 @@ require (
 	github.com/hashicorp/go-multierror v1.1.0
 	github.com/hashicorp/go-plugin v1.3.0 // indirect
 	github.com/hashicorp/go-retryablehttp v0.6.7 // indirect
+	github.com/hashicorp/hcl/v2 v2.8.0
 	github.com/hashicorp/raft v1.1.2 // indirect
 	github.com/hashicorp/vault v1.2.1-0.20200522144850-6f72d4ff250f
 	github.com/hashicorp/vault-plugin-auth-gcp v0.8.0 // indirect
diff --git a/go.sum b/go.sum
index d30523c..d857899 100644
--- a/go.sum
+++ b/go.sum
@@ -155,6 +155,8 @@ github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrU
 github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g=
 github.com/abdullin/seq v0.0.0-20160510034733-d5467c17e7af/go.mod h1:5Jv4cbFiHJMsVxt52+i0Ha45fjshj6wxYr1r19tB9bw=
 github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c=
+github.com/agext/levenshtein v1.2.1 h1:QmvMAjj2aEICytGiWzmxoE0x2KZvE0fvmqMOfy2tjT8=
+github.com/agext/levenshtein v1.2.1/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558=
 github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM=
 github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
@@ -171,6 +173,11 @@ github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kd
 github.com/apache/arrow/go/arrow v0.0.0-20200601151325-b2287a20f230/go.mod h1:QNYViu/X0HXDHw7m3KXzWSVXIbfUvJqBFe6Gj8/pYA0=
 github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
 github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
+github.com/apparentlymart/go-dump v0.0.0-20180507223929-23540a00eaa3/go.mod h1:oL81AME2rN47vu18xqj1S1jPIPuN7afo62yKTNn3XMM=
+github.com/apparentlymart/go-textseg v1.0.0 h1:rRmlIsPEEhUTIKQb7T++Nz/A5Q6C9IuX2wFoYVvnCs0=
+github.com/apparentlymart/go-textseg v1.0.0/go.mod h1:z96Txxhf3xSFMPmb5X/1W05FF/Nj9VFpLOpjS5yuumk=
+github.com/apparentlymart/go-textseg/v12 v12.0.0 h1:bNEQyAGak9tojivJNkoqWErVCQbjdL7GzRt3F8NvfJ0=
+github.com/apparentlymart/go-textseg/v12 v12.0.0/go.mod h1:S/4uRK2UtaQttw1GenVJEynmyUenKwP++x/+DdGV/Ec=
 github.com/apple/foundationdb/bindings/go v0.0.0-20190411004307-cd5c9d91fad2/go.mod h1:OMVSB21p9+xQUIqlGizHPZfjK+SHws1ht+ZytVDoz9U=
 github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
 github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
@@ -400,6 +407,7 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me
 github.com/go-test/deep v1.0.1/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
 github.com/go-test/deep v1.0.2-0.20181118220953-042da051cf31/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
 github.com/go-test/deep v1.0.2/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
+github.com/go-test/deep v1.0.3/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
 github.com/go-test/deep v1.0.6 h1:UHSEyLZUwX9Qoi99vVwvewiMC8mM2bf7XEM2nqvzEn8=
 github.com/go-test/deep v1.0.6/go.mod h1:QV8Hv/iy04NyLBxAdO9njL0iVPN1S4d/A3NVv1V36o8=
 github.com/go-yaml/yaml v2.1.0+incompatible h1:RYi2hDdss1u4YE7GwixGzWwVo47T8UQwnTLB6vQiq+o=
@@ -441,6 +449,7 @@ github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt
 github.com/golang/mock v1.4.4 h1:l75CXGRSwbaYNpl/Z2X1XIIAMSCquvXgpVZDhwEIJsc=
 github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
 github.com/golang/protobuf v0.0.0-20161109072736-4bd1920723d7/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.1.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
@@ -627,6 +636,8 @@ github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+l
 github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
+github.com/hashicorp/hcl/v2 v2.8.0 h1:iHLEAsNDp3N2MtqroP1wf0nF/zB2+McHN5YCzwqIm80=
+github.com/hashicorp/hcl/v2 v2.8.0/go.mod h1:bQTN5mpo+jewjJgh8jr0JUguIi7qPHUF6yIfAEN3jqY=
 github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
 github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
 github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
@@ -851,6 +862,8 @@ github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4=
+github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
 github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y=
 github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
 github.com/lestrrat-go/jwx v0.9.0/go.mod h1:iEoxlYfZjvoGpuWwxUz+eR5e6KTJGsaRcy/YNA/UnBk=
@@ -918,6 +931,8 @@ github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.
 github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
 github.com/mitchellh/go-testing-interface v1.14.1 h1:jrgshOhYAUVNMAJiKbEu7EqAwgJJ2JqpQmpLJOu07cU=
 github.com/mitchellh/go-testing-interface v1.14.1/go.mod h1:gfgS7OtZj6MA4U1UrDRp04twqAjfvlZyCfX3sDjEym8=
+github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo=
+github.com/mitchellh/go-wordwrap v1.0.0 h1:6GlHJ/LTGMrIJbwgdqdl2eEH8o+Exx/0m8ir9Gns0u4=
 github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo=
 github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
 github.com/mitchellh/gox v1.0.1/go.mod h1:ED6BioOGXMswlXa2zxfh/xdd5QhwYliBFn9V18Ap4z4=
@@ -1130,6 +1145,8 @@ github.com/sean-/conswriter v0.0.0-20180208195008-f5ae3917a627/go.mod h1:7zjs06q
 github.com/sean-/pager v0.0.0-20180208200047-666be9bf53b5/go.mod h1:BeybITEsBEg6qbIiqJ6/Bqeq25bCLbL7YFmpaFfJDuM=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I=
 github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
+github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
+github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/sethvargo/go-envconfig v0.3.2 h1:277Lb2iTpUZjUZu1qLoLa/aetwvtZbKh8wNWXmc6dSk=
 github.com/sethvargo/go-envconfig v0.3.2/go.mod h1:XZ2JRR7vhlBEO5zMmOpLgUhgYltqYqq4d4tKagtPUv0=
 github.com/sethvargo/go-gcpkms v0.1.0 h1:pyjDLqLwpk9pMjDSTilPpaUjgP1AfSjX9WGzitZwGUY=
@@ -1178,6 +1195,7 @@ github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb6
 github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.1-0.20171106142849-4c012f6dcd95/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.2/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
@@ -1224,6 +1242,7 @@ github.com/ugorji/go/codec v1.1.8/go.mod h1:X00B19HDtwvKbQY2DcYjvZxKQp8mzrJoQ6Eg
 github.com/ulikunitz/xz v0.5.6/go.mod h1:2bypXElzHzzJZwzH67Y6wb67pO62Rzfn7BSiF4ABRW8=
 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
 github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
+github.com/vmihailenco/msgpack v3.3.3+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk=
 github.com/xanzy/go-gitlab v0.15.0/go.mod h1:8zdQa/ri1dfn8eS3Ir1SyfvOKlw7WBJ8DVThkpGiXrs=
 github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk=
 github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
@@ -1236,6 +1255,8 @@ github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/zclconf/go-cty v1.2.0 h1:sPHsy7ADcIZQP3vILvTjrh74ZA175TFP5vqiNK1UmlI=
+github.com/zclconf/go-cty v1.2.0/go.mod h1:hOPWgoHbaTUnI5k4D2ld+GRpFJSCe6bCM7m1q/N4PQ8=
 github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q=
 gitlab.com/nyarla/go-crypt v0.0.0-20160106005555-d9a5dc2b789b/go.mod h1:T3BPAOm2cqquPa0MKWeNkmOM5RQsRhkrwMWonFMN7fE=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
@@ -1287,6 +1308,7 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
 golang.org/x/crypto v0.0.0-20190325154230-a5d413f7728c/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
 golang.org/x/crypto v0.0.0-20190418165655-df01cb2cc480/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE=
+golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
@@ -1342,6 +1364,7 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180530234432-1e491301e022/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180811021610-c39426892332/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
diff --git a/terraform/alerting/alerts.tf b/terraform/alerting/alerts.tf
index de8e9e2..40ede79 100644
--- a/terraform/alerting/alerts.tf
+++ b/terraform/alerting/alerts.tf
@@ -13,24 +13,29 @@
 # limitations under the License.
 
 locals {
-  p50_latency_thresholds_in_seconds = {
+  p50_latency_thresholds = {
     export         = 600
     cleanup-export = 60
     generate       = 30
   }
-  p50_latency_thresholds_in_seconds_default = 10
+  p50_latency_thresholds_default = 10
 
   p50_latency_condition = join("\n|| ", concat(
     [
-      for k, v in local.p50_latency_thresholds_in_seconds :
-      "(resource.service_name == '${k}' && val > ${v * 1000} 'ms')"
+      for k, v in local.p50_latency_thresholds:
+      "(resource.service_name == '${k}' && val > ${v} 's')"
     ],
     [
-      "(val > ${local.p50_latency_thresholds_in_seconds_default * 1000} 'ms')"
+      "(val > ${local.p50_latency_thresholds_default} 's')"
     ]
   ))
 }
 
+locals {
+  playbook_prefix = "https://github.com/google/exposure-notifications-server/blob/main/docs/playbooks/alerts"
+}
+
+
 resource "google_monitoring_alert_policy" "LatencyTooHigh" {
   combiner     = "OR"
   display_name = "LatencyTooHigh"
@@ -56,9 +61,7 @@ resource "google_monitoring_alert_policy" "LatencyTooHigh" {
     }
   }
   documentation {
-    content   = <<-EOT
-    Our latency is too high for one or more Cloud Run services!
-    EOT
+    content   = "${local.playbook_prefix}/LatencyTooHigh.md"
     mime_type = "text/markdown"
   }
   notification_channels = [
diff --git a/terraform/dos.tf b/terraform/dos.tf
new file mode 100644
index 0000000..626b8e0
--- /dev/null
+++ b/terraform/dos.tf
@@ -0,0 +1,117 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This file contains the Cloud Armor configs.
+
+resource "google_compute_security_policy" "cloud-armor" {
+  name = "cloud-armor-protection"
+  rule {
+    action      = "deny(403)"
+    description = "XSS protection"
+    match {
+      expr { expression = "evaluatePreconfiguredExpr('xss-stable')" }
+    }
+    preview  = false
+    priority = 10
+  }
+  rule {
+    action      = "deny(403)"
+    description = "SQL Injection protection"
+    match {
+      expr { expression = "evaluatePreconfiguredExpr('sqli-stable')" }
+    }
+    preview  = false
+    priority = 20
+  }
+  rule {
+    action      = "deny(403)"
+    description = "Remote Code Execution protection"
+    match {
+      expr { expression = "evaluatePreconfiguredExpr('rce-stable')" }
+    }
+    preview  = false
+    priority = 30
+  }
+  // Recommended action when we want to protect the service during incident:
+  // 1. See if there's any rule below can be used as is. Use that.
+  // 2. Otherwise check
+  //    https://cloud.google.com/armor/docs/rules-language-reference and see if
+  //    there's other attributes that you can use.
+  // 3. Run `terraform apply` with "preview=true" first.
+  // 4. Go to "Monitoring - Dashboards" and choose "Network Security Policies".
+  //    Check the "Previewd Requests" graph at the bottom and see how many
+  //    traffic would be blocked if "preview=false". Ensure the rule doesn't
+  //    block all traffic by accident. If it does block too many traffics, go
+  //    back and review the expression.
+  // 5. Run `terraform apply` again with "preview=false".
+  //
+  // Block certain User-Agent
+  // ========================
+  // rule {
+  //   action      = "deny(403)"
+  //   description = "Block User-Agent FOO"
+  //   match {
+  //     expr {
+  //       expression = <<-EOT
+  //         has(request.headers['user-agent']) &&
+  //         request.headers.['user-agent'].matches('FOO')
+  //       EOT
+  //     }
+  //   }
+  //   preview  = true
+  //   priority = 40
+  // }
+  //
+  // Block IP ranges
+  // ===============
+  // rule {
+  //   action      = "deny(403)"
+  //   description = "Block IP Range FOO"
+  //   match {
+  //     expr {
+  //       expression = <<-EOT
+  //         inIPRange(origin.ip, '1.2.3.0/24')
+  //       EOT
+  //     }
+  //   }
+  //   preview  = true
+  //   priority = 50
+  // }
+  //
+  // Block access to a certain URL path
+  // ==================================
+  // rule {
+  //   action      = "deny(403)"
+  //   description = "Block path FOO"
+  //   match {
+  //     expr {
+  //       expression = <<-EOT
+  //         request.path.matches('FOO')
+  //       EOT
+  //     }
+  //   }
+  //   preview  = true
+  //   priority = 50
+  // }
+  rule {
+    action      = "allow"
+    description = "Default rule, higher priority overrides it"
+    match {
+      config { src_ip_ranges = ["*"] }
+      versioned_expr = "SRC_IPS_V1"
+    }
+    preview  = false
+    priority = 2147483647
+  }
+}
diff --git a/terraform/service_debugger.tf b/terraform/service_debugger.tf
index 65ac474..dc34a17 100644
--- a/terraform/service_debugger.tf
+++ b/terraform/service_debugger.tf
@@ -164,6 +164,7 @@ resource "google_compute_backend_service" "debugger" {
   backend {
     group = google_compute_region_network_endpoint_group.debugger[0].id
   }
+  security_policy = google_compute_security_policy.cloud-armor.name
 }
 
 output "debugger_urls" {
diff --git a/terraform/service_exposure.tf b/terraform/service_exposure.tf
index 70932ca..54ebf86 100644
--- a/terraform/service_exposure.tf
+++ b/terraform/service_exposure.tf
@@ -170,6 +170,7 @@ resource "google_compute_backend_service" "exposure" {
   backend {
     group = google_compute_region_network_endpoint_group.exposure[0].id
   }
+  security_policy = google_compute_security_policy.cloud-armor.name
 }
 
 output "exposure_urls" {
diff --git a/terraform/service_federationout.tf b/terraform/service_federationout.tf
index aafeb22..f5e8462 100644
--- a/terraform/service_federationout.tf
+++ b/terraform/service_federationout.tf
@@ -154,6 +154,7 @@ resource "google_compute_backend_service" "federationout" {
   backend {
     group = google_compute_region_network_endpoint_group.federationout[0].id
   }
+  security_policy = google_compute_security_policy.cloud-armor.name
 }
 
 output "federationout_urls" {


Test patch:
diff --git a/docs/playbooks/playbook_test.go b/docs/playbooks/playbook_test.go
new file mode 100644
index 0000000..5c8104e
--- /dev/null
+++ b/docs/playbooks/playbook_test.go
@@ -0,0 +1,125 @@
+// Copyright 2020 Google LLC
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package main
+
+import (
+	"path/filepath"
+	"sort"
+	"testing"
+
+	"github.com/hashicorp/hcl/v2"
+	"github.com/hashicorp/hcl/v2/gohcl"
+	"github.com/hashicorp/hcl/v2/hclparse"
+)
+
+const (
+	alertTF = "../../terraform/alerting/alerts.tf"
+
+	// Ignore files not starting with an upper case letter.
+	playbooksGlob = "./alerts/[A-Z]*.md"
+)
+
+var alertPolicySchema = hcl.BodySchema{
+	Attributes: []hcl.AttributeSchema{
+		{Name: "display_name", Required: true},
+	},
+}
+
+type resource struct {
+	Type      string   `hcl:"type,label"`
+	Name      string   `hcl:"name,label"`
+	Remaining hcl.Body `hcl:",remain"`
+}
+
+type config struct {
+	Resources []*resource `hcl:"resource,block"`
+	Remaining hcl.Body    `hcl:",remain"`
+}
+
+func allPlaybookNames(t *testing.T) []string {
+	files, err := filepath.Glob(playbooksGlob)
+	if err != nil {
+		t.Fatalf("failed to find all playbooks: %s", err)
+	}
+	var ret []string
+	for _, x := range files {
+		basename := filepath.Base(x)
+		ret = append(ret, basename[:len(basename)-len(".md")])
+	}
+	return ret
+}
+
+func allAlertNames(t *testing.T) []string {
+	hcl, diag := hclparse.NewParser().ParseHCLFile(alertTF)
+	if diag.HasErrors() {
+		t.Fatalf("failed to parse alerts.tf: %v", diag)
+	}
+	var c config
+	diag = gohcl.DecodeBody(hcl.Body, nil, &c)
+	if diag.HasErrors() {
+		t.Fatalf("failed to decode body: %v", diag)
+	}
+	var ret []string
+	for _, res := range c.Resources {
+		if res.Type != "google_monitoring_alert_policy" {
+			continue
+		}
+		content, _, diag := res.Remaining.PartialContent(&alertPolicySchema)
+		if diag.HasErrors() {
+			t.Fatalf("failed to get attributes from block body: %v", diag)
+		}
+		v, diag := content.Attributes["display_name"].Expr.Value(nil)
+		if diag.HasErrors() {
+			t.Fatalf("failed to evaluate value of display_name in an alert policy block: %v", diag)
+		}
+		ret = append(ret, v.AsString())
+	}
+	return ret
+}
+
+func setDiff(xs, ys []string) []string {
+	s1 := make(map[string]bool)
+	s2 := make(map[string]bool)
+	for _, s := range xs {
+		s1[s] = true
+	}
+	for _, s := range ys {
+		s2[s] = true
+	}
+	for k := range s1 {
+		if s2[k] {
+			delete(s1, k)
+		}
+	}
+	var ret []string
+	for k := range s1 {
+		ret = append(ret, k)
+	}
+	sort.Strings(ret)
+	return ret
+}
+
+// Test to ensure every playbook has a corresponding alert, and every alert has
+// a corresponding laybook.
+func TestPlaybooks(t *testing.T) {
+	playbooks := allPlaybookNames(t)
+	alerts := allAlertNames(t)
+	for _, x := range setDiff(playbooks, alerts) {
+		t.Errorf("Missing alert for playbook %q.", x)
+	}
+	for _, x := range setDiff(alerts, playbooks) {
+		t.Errorf("Missing playbook for alert %q.", x)
+	}
+}
diff --git a/internal/export/worker_test.go b/internal/export/worker_test.go
index e40e029..9d79c48 100644
--- a/internal/export/worker_test.go
+++ b/internal/export/worker_test.go
@@ -119,7 +119,7 @@ func TestEnsureMinExposures(t *testing.T) {
 	if err != nil {
 		t.Fatalf("unexpected error: %v", err)
 	}
-	if len(exposures) < numKeys || len(exposures) > numKeys+variance {
+	if len(exposures) < numKeys || len(exposures) > (numKeys+variance+inputSize-1) {
 		t.Errorf("wrong number of exposures, want: >=%v and <=%v, got: %v", numKeys, numKeys+variance, len(exposures))
 	}
 	if l, exp := len(generated), numKeys-inputSize; l < exp {