Repository: google/exposure-notifications-server
Pull request #1523 (2021-05-20 23:01:43)

release key server stats 48 hours after end of UTC day even if not en…
…ough entries

## Proposed Changes

* introduce new config for stats embargo period
* if stats embargo period has passed (measured from end of UTC day) and there are not enough publish requests, release the stats (that are currently held back)

**Release Note**

```release-note
NOTICE: This changes the behavior of the stats API. Introduces an embargo period for publish stats. If a health authority has low publish request, below the STATS_UPLOAD_MINIMUM, this allows you to release their stats after a certain embargo period has been met.
```

mikehelmick: Please do not merge - let me do so when ready

Patch:
diff --git a/internal/publish/config.go b/internal/publish/config.go
index 1c4db98..c03b5c1 100644
--- a/internal/publish/config.go
+++ b/internal/publish/config.go
@@ -108,9 +108,12 @@ type Config struct {
 	// Publish stats API config
 	// Minimum number of publish requests that need to be present to see stats for a given day.
 	// If the minimum is not met, that day is not revealed or shown in aggregates.
-	StatsUploadMinimum           int64 `env:"STATS_UPLOAD_MINIMUM, default=10"`
-	StatsResponsePaddingMinBytes int64 `env:"RESPONSE_PADDING_MIN_BYTES, default=2048"`
-	StatsResponsePaddingRange    int64 `env:"RESPONSE_PADDING_RANGE, default=1024"`
+	StatsUploadMinimum int64 `env:"STATS_UPLOAD_MINIMUM, default=10"`
+	// Allow release of a day's stats after this much time has passed (measured from end of day).
+	// Set to <= 0 to disable this feature.
+	StatsEmbargoPeriod           time.Duration `env:"STATS_EMBARGO_PERIOD, default=48h"`
+	StatsResponsePaddingMinBytes int64         `env:"RESPONSE_PADDING_MIN_BYTES, default=2048"`
+	StatsResponsePaddingRange    int64         `env:"RESPONSE_PADDING_RANGE, default=1024"`
 }
 
 func (c *Config) MaintenanceMode() bool {
diff --git a/internal/publish/model/stats.go b/internal/publish/model/stats.go
index 315ea06..482f39a 100644
--- a/internal/publish/model/stats.go
+++ b/internal/publish/model/stats.go
@@ -65,7 +65,10 @@ type HealthAuthorityStats struct {
 // time indicates the cutoff point for inclusion.
 // The dayThreshold indicates how many entries are needed to include a given day.
 // Day boundaries are all in UTC.
-func ReduceStats(hourly []*HealthAuthorityStats, onlyBefore time.Time, dayThreshold int64) []*verifyapi.StatsDay {
+func ReduceStats(hourly []*HealthAuthorityStats, onlyBefore time.Time, dayThreshold int64, embargoPeriod time.Duration) []*verifyapi.StatsDay {
+	embargoRelease := embargoPeriod > 0
+	now := time.Now().UTC()
+
 	days := make(map[time.Time]*verifyapi.StatsDay)
 	// Combine the hours into days based on same UTC midnight time.
 	for _, hour := range hourly {
@@ -102,7 +105,10 @@ func ReduceStats(hourly []*HealthAuthorityStats, onlyBefore time.Time, dayThresh
 	// Bring the map back to an array
 	result := make([]*verifyapi.StatsDay, 0, len(days))
 	for _, day := range days {
-		if day.PublishRequests.Total() < dayThreshold {
+		endOfDay := timeutils.UTCMidnight(day.Day.Add(24 * time.Hour))
+		embargoOver := endOfDay.Add(embargoPeriod).Before(now)
+		okToShow := day.PublishRequests.Total() >= dayThreshold || (embargoRelease && embargoOver)
+		if !okToShow {
 			continue
 		}
 		result = append(result, day)
diff --git a/internal/publish/stats.go b/internal/publish/stats.go
index 423f5b0..7df5130 100644
--- a/internal/publish/stats.go
+++ b/internal/publish/stats.go
@@ -103,7 +103,7 @@ func (s *Server) handleMetricsRequest(ctx context.Context, bearerToken string, r
 	onlyBefore := time.Now().UTC().Truncate(time.Hour)
 
 	// Combine days - this also filters things that are "too new" and days that don't meet the threshold.
-	response.Days = model.ReduceStats(stats, onlyBefore, s.config.StatsUploadMinimum)
+	response.Days = model.ReduceStats(stats, onlyBefore, s.config.StatsUploadMinimum, s.config.StatsEmbargoPeriod)
 
 	// return
 	return response, http.StatusOK


Test patch:
diff --git a/internal/publish/model/stats_test.go b/internal/publish/model/stats_test.go
index 22c7850..15553e4 100644
--- a/internal/publish/model/stats_test.go
+++ b/internal/publish/model/stats_test.go
@@ -152,14 +152,25 @@ func minPadSlice(s []int64, size int) []int64 {
 func TestReduce(t *testing.T) {
 	t.Parallel()
 
-	hour := timeutils.UTCMidnight(time.Now().UTC()).Add(-48 * time.Hour)
+	hour := timeutils.UTCMidnight(time.Now().UTC()).Add(-72 * time.Hour)
 	startTime := hour
 
 	input := []*HealthAuthorityStats{
-		// two entries from 2 days ago
+		// one entry from 3 days ago
 		{
 			HealthAuthorityID: 42,
 			Hour:              hour,
+			PublishCount:      []int64{1, 1, 1},
+			TEKCount:          42,
+			RevisionCount:     0,
+			OldestTekDays:     []int64{0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 2, 0, 0, 10, 1},
+			OnsetAgeDays:      minPadSlice([]int64{1, 1, 1}, StatsMaxOnsetDays+1),
+			MissingOnset:      1,
+		},
+		// two entries from 2 days ago
+		{
+			HealthAuthorityID: 42,
+			Hour:              hour.Add(24 * time.Hour),
 			PublishCount:      []int64{1, 10, 4},
 			TEKCount:          112,
 			RevisionCount:     0,
@@ -169,7 +180,7 @@ func TestReduce(t *testing.T) {
 		},
 		{
 			HealthAuthorityID: 42,
-			Hour:              hour.Add(time.Hour),
+			Hour:              hour.Add(25 * time.Hour),
 			PublishCount:      []int64{0, 5, 6},
 			TEKCount:          65,
 			RevisionCount:     0,
@@ -180,7 +191,7 @@ func TestReduce(t *testing.T) {
 		// one entry from 1 days ago, but not enough uploads to be shown
 		{
 			HealthAuthorityID: 42,
-			Hour:              hour.Add(24 * time.Hour),
+			Hour:              hour.Add(48 * time.Hour),
 			PublishCount:      []int64{1, 1, 1},
 			TEKCount:          42,
 			RevisionCount:     0,
@@ -191,7 +202,7 @@ func TestReduce(t *testing.T) {
 		// two entries from today, but one shouldn't be shown (time hasn't lapsed yet)
 		{
 			HealthAuthorityID: 42,
-			Hour:              hour.Add(48 * time.Hour),
+			Hour:              hour.Add(72 * time.Hour),
 			PublishCount:      []int64{0, 187, 294},
 			TEKCount:          6734,
 			RevisionCount:     0,
@@ -201,7 +212,7 @@ func TestReduce(t *testing.T) {
 		},
 		{ // This record should be held back.
 			HealthAuthorityID: 42,
-			Hour:              hour.Add(49 * time.Hour),
+			Hour:              hour.Add(73 * time.Hour),
 			PublishCount:      []int64{0, 5, 6},
 			TEKCount:          65,
 			RevisionCount:     0,
@@ -214,6 +225,19 @@ func TestReduce(t *testing.T) {
 	want := []*verifyapi.StatsDay{
 		{
 			Day: startTime,
+			PublishRequests: verifyapi.PublishRequests{
+				UnknownPlatform: 1,
+				Android:         1,
+				IOS:             1,
+			},
+			TotalTEKsPublished:        42,
+			RevisionRequests:          0,
+			TEKAgeDistribution:        []int64{0, 0, 0, 0, 0, 1, 0, 0, 1, 0, 0, 2, 0, 0, 10, 1},
+			OnsetToUploadDistribution: minPadSlice([]int64{1, 1, 1}, StatsMaxOnsetDays+1),
+			RequestsMissingOnsetDate:  1,
+		},
+		{
+			Day: startTime.Add(24 * time.Hour),
 			PublishRequests: verifyapi.PublishRequests{
 				UnknownPlatform: 1,
 				Android:         15,
@@ -226,7 +250,7 @@ func TestReduce(t *testing.T) {
 			RequestsMissingOnsetDate:  1,
 		},
 		{
-			Day: startTime.Add(48 * time.Hour),
+			Day: startTime.Add(72 * time.Hour),
 			PublishRequests: verifyapi.PublishRequests{
 				UnknownPlatform: 0,
 				Android:         187,
@@ -240,7 +264,7 @@ func TestReduce(t *testing.T) {
 		},
 	}
 
-	got := ReduceStats(input, hour.Add(49*time.Hour), 10)
+	got := ReduceStats(input, hour.Add(73*time.Hour), 10, 48*time.Hour)
 
 	if diff := cmp.Diff(want, got); diff != "" {
 		t.Fatalf("mismatch (-want, +got):\n%s", diff)