Repository: google/exposure-notifications-server
Pull request #1443 (2021-03-09 19:05:32)

add ability for export-importer to import an export file w/ travler=true

## Proposed Changes

* export-importer can set traveller status to true (for the whole import)

**Release Note**

```release-note
export-importer can set traveller status to true (for the whole import)
```


Patch:
diff --git a/internal/exportimport/database/exportimport.go b/internal/exportimport/database/exportimport.go
index f111813..4b7e7a6 100644
--- a/internal/exportimport/database/exportimport.go
+++ b/internal/exportimport/database/exportimport.go
@@ -44,7 +44,7 @@ func (db *ExportImportDB) GetConfig(ctx context.Context, id int64) (*model.Expor
 	if err := db.db.InTx(ctx, pgx.RepeatableRead, func(tx pgx.Tx) error {
 		row := tx.QueryRow(ctx, `
 			SELECT
-				id, index_file, export_root, region, from_timestamp, thru_timestamp
+				id, index_file, export_root, region, traveler, from_timestamp, thru_timestamp
 			FROM
 				exportimport
 			WHERE
@@ -71,7 +71,7 @@ func (db *ExportImportDB) ListConfigs(ctx context.Context) ([]*model.ExportImpor
 	if err := db.db.InTx(ctx, pgx.ReadCommitted, func(tx pgx.Tx) error {
 		rows, err := tx.Query(ctx, `
 			SELECT
-				id, index_file, export_root, region, from_timestamp, thru_timestamp
+				id, index_file, export_root, region, traveler, from_timestamp, thru_timestamp
 			FROM
 				exportimport
 			ORDER BY id ASC
@@ -107,7 +107,7 @@ func (db *ExportImportDB) ActiveConfigs(ctx context.Context) ([]*model.ExportImp
 	if err := db.db.InTx(ctx, pgx.ReadCommitted, func(tx pgx.Tx) error {
 		rows, err := tx.Query(ctx, `
 			SELECT
-				id, index_file, export_root, region, from_timestamp, thru_timestamp
+				id, index_file, export_root, region, traveler, from_timestamp, thru_timestamp
 			FROM
 				exportimport
 			WHERE
@@ -146,7 +146,7 @@ func scanOneConfig(row pgx.Row) (*model.ExportImport, error) {
 		thru *time.Time
 	)
 
-	if err := row.Scan(&m.ID, &m.IndexFile, &m.ExportRoot, &m.Region, &m.From, &thru); err != nil {
+	if err := row.Scan(&m.ID, &m.IndexFile, &m.ExportRoot, &m.Region, &m.Traveler, &m.From, &thru); err != nil {
 		return nil, err
 	}
 	if thru != nil {
@@ -166,11 +166,11 @@ func (db *ExportImportDB) AddConfig(ctx context.Context, ei *model.ExportImport)
 		row := tx.QueryRow(ctx, `
 			INSERT INTO
 			ExportImport
-				(index_file, export_root, region, from_timestamp, thru_timestamp)
+				(index_file, export_root, region, traveler, from_timestamp, thru_timestamp)
 			VALUES
-				($1, $2, $3, $4, $5)
+				($1, $2, $3, $4, $5, $6)
 			RETURNING id
-		`, ei.IndexFile, ei.ExportRoot, ei.Region, ei.From, ei.Thru)
+		`, ei.IndexFile, ei.ExportRoot, ei.Region, ei.Traveler, ei.From, ei.Thru)
 
 		if err := row.Scan(&ei.ID); err != nil {
 			return fmt.Errorf("fetching exportimport.ID: %w", err)
@@ -191,9 +191,9 @@ func (db *ExportImportDB) UpdateConfig(ctx context.Context, c *model.ExportImpor
 			UPDATE
 				ExportImport
 			SET
-				index_file = $1, export_root = $2, region = $3, from_timestamp = $4, thru_timestamp = $5
-			WHERE id = $6
-		`, c.IndexFile, c.ExportRoot, c.Region, from, c.Thru, c.ID)
+				index_file = $1, export_root = $2, region = $3, traveler = $4, from_timestamp = $5, thru_timestamp = $6
+			WHERE id = $7
+		`, c.IndexFile, c.ExportRoot, c.Region, c.Traveler, from, c.Thru, c.ID)
 		if err != nil {
 			return fmt.Errorf("failed to update export importer config: %w", err)
 		}
diff --git a/internal/exportimport/import.go b/internal/exportimport/import.go
index 8971583..f4ff687 100644
--- a/internal/exportimport/import.go
+++ b/internal/exportimport/import.go
@@ -146,6 +146,7 @@ func (s *Server) ImportExportFile(ctx context.Context, ir *ImportRequest) (*Impo
 	exKeyTransform := transformer{
 		appPackageName: s.config.ImportAPKName,
 		importRegions:  []string{ir.exportImport.Region},
+		traveler:       ir.exportImport.Traveler,
 		batchTime:      time.Now().UTC().Truncate(s.config.CreatedAtTruncateWindow),
 		truncateWindow: s.config.CreatedAtTruncateWindow,
 		exportImportID: ir.exportImport.ID,
@@ -223,6 +224,7 @@ func (s *Server) insertAndReviseKeys(ctx context.Context, mode string, exposures
 type transformer struct {
 	appPackageName     string
 	importRegions      []string
+	traveler           bool
 	batchTime          time.Time
 	truncateWindow     time.Duration
 	exportImportID     int64
@@ -244,6 +246,7 @@ func (t *transformer) transform(keys []*exportproto.TemporaryExposureKey) ([]*pu
 		// Fill in items that are specific to this import.
 		exp.AppPackageName = t.appPackageName
 		exp.Regions = t.importRegions
+		exp.Traveler = t.traveler
 		exp.CreatedAt = t.batchTime
 		exp.LocalProvenance = false
 		exp.ExportImportID = &t.exportImportID
diff --git a/internal/exportimport/model/exportimport.go b/internal/exportimport/model/exportimport.go
index f18637d..6df0d4f 100644
--- a/internal/exportimport/model/exportimport.go
+++ b/internal/exportimport/model/exportimport.go
@@ -28,6 +28,7 @@ type ExportImport struct {
 	IndexFile  string
 	ExportRoot string
 	Region     string
+	Traveler   bool
 	From       time.Time
 	Thru       *time.Time
 }
diff --git a/migrations/000079_ExportImportTraveler.down.sql b/migrations/000079_ExportImportTraveler.down.sql
new file mode 100644
index 0000000..98bf24f
--- /dev/null
+++ b/migrations/000079_ExportImportTraveler.down.sql
@@ -0,0 +1,20 @@
+-- Copyright 2021 Google LLC
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--      http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+BEGIN;
+
+ALTER TABLE exportimport
+    DROP COLUMN traveler;
+
+END;
diff --git a/migrations/000079_ExportImportTraveler.up.sql b/migrations/000079_ExportImportTraveler.up.sql
new file mode 100644
index 0000000..25c3e5c
--- /dev/null
+++ b/migrations/000079_ExportImportTraveler.up.sql
@@ -0,0 +1,25 @@
+-- Copyright 2021 Google LLC
+--
+-- Licensed under the Apache License, Version 2.0 (the "License");
+-- you may not use this file except in compliance with the License.
+-- You may obtain a copy of the License at
+--
+--      http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+
+BEGIN;
+
+ALTER TABLE exportimport
+  ADD COLUMN traveler BOOL DEFAULT false;
+
+UPDATE exportimport SET traveler=false;
+
+ALTER TABLE exportimport
+  ALTER COLUMN traveler SET NOT NULL;
+
+END;


Test patch:
diff --git a/internal/exportimport/database/exportimport_test.go b/internal/exportimport/database/exportimport_test.go
index 6f606e7..521c91c 100644
--- a/internal/exportimport/database/exportimport_test.go
+++ b/internal/exportimport/database/exportimport_test.go
@@ -50,6 +50,7 @@ func TestAddGetUpdateExportConfig(t *testing.T) {
 			IndexFile:  "https://myserver2/exports/index.txt",
 			ExportRoot: "https://myserver2/",
 			Region:     "US",
+			Traveler:   true,
 			From:       fromTime.Add(time.Hour),
 			Thru:       nil,
 		},
diff --git a/internal/exportimport/import_test.go b/internal/exportimport/import_test.go
index ad94514..2468e19 100644
--- a/internal/exportimport/import_test.go
+++ b/internal/exportimport/import_test.go
@@ -59,82 +59,106 @@ func (k *keyGenerator) fakeExposureKey(t testing.TB) []byte {
 func TestTransform(t *testing.T) {
 	t.Parallel()
 
-	ctx := project.TestContext(t)
-	logger := logging.FromContext(ctx)
-
-	gen := &keyGenerator{}
-
-	// batch time, one hour after midnight.
-	// Will be used as the offset when creating TEKs.
-	batchTime := timeutils.UTCMidnight(time.Now().UTC()).Add(time.Hour)
-
-	sameDay := timeutils.UTCMidnight(batchTime)
-
-	input := []*exportproto.TemporaryExposureKey{
-		{ // Valid key, fully expired, missing report type and transmission risk
-			KeyData:                    gen.fakeExposureKey(t),
-			RollingStartIntervalNumber: proto.Int32(pubmodel.IntervalNumber(sameDay.Add(-24 * time.Hour))),
-			RollingPeriod:              proto.Int32(144),
-		},
-		{ // same day key, should be future dated
-			KeyData:                    gen.fakeExposureKey(t),
-			RollingStartIntervalNumber: proto.Int32(pubmodel.IntervalNumber(sameDay)),
-			RollingPeriod:              proto.Int32(144),
-		},
-	}
-
-	appPackageName := "import-export-test"
-	regions := []string{"US"}
-	settings := &transformer{
-		appPackageName: appPackageName,
-		importRegions:  regions,
-		batchTime:      batchTime,
-		truncateWindow: time.Hour,
-		exportImportID: 7,
-		importFileID:   42,
-		exportImportConfig: &pubmodel.ExportImportConfig{
-			DefaultReportType:         verifyapi.ReportTypeConfirmed,
-			BackfillSymptomOnset:      true,
-			BackfillSymptomOnsetValue: 10,
-			MaxSymptomOnsetDays:       28,
-			AllowClinical:             false,
-			AllowRevoked:              false,
-		},
-		logger: logger,
-	}
-
-	got, _ := settings.transform(input)
-
-	want := []*pubmodel.Exposure{
+	cases := []struct {
+		name     string
+		traveler bool
+	}{
 		{
-			ExposureKey:           input[0].KeyData,
-			TransmissionRisk:      2,
-			AppPackageName:        appPackageName,
-			Regions:               regions,
-			IntervalNumber:        input[0].GetRollingStartIntervalNumber(),
-			IntervalCount:         input[0].GetRollingPeriod(),
-			CreatedAt:             batchTime,
-			ExportImportID:        &settings.exportImportID,
-			ImportFileID:          &settings.importFileID,
-			ReportType:            verifyapi.ReportTypeConfirmed,
-			DaysSinceSymptomOnset: &settings.exportImportConfig.BackfillSymptomOnsetValue,
+			name:     "not_travler",
+			traveler: false,
 		},
-		{ // Key hasn't yet expired, so it is future dated into tomorrow.
-			ExposureKey:           input[1].KeyData,
-			TransmissionRisk:      2,
-			AppPackageName:        appPackageName,
-			Regions:               regions,
-			IntervalNumber:        input[1].GetRollingStartIntervalNumber(),
-			IntervalCount:         input[1].GetRollingPeriod(),
-			CreatedAt:             timeutils.UTCMidnight(batchTime.Add(24 * time.Hour)).Add(settings.truncateWindow),
-			ExportImportID:        &settings.exportImportID,
-			ImportFileID:          &settings.importFileID,
-			ReportType:            verifyapi.ReportTypeConfirmed,
-			DaysSinceSymptomOnset: &settings.exportImportConfig.BackfillSymptomOnsetValue,
+		{
+			name:     "traveler",
+			traveler: true,
 		},
 	}
 
-	if diff := cmp.Diff(want, got, cmpopts.IgnoreUnexported(pubmodel.Exposure{})); diff != "" {
-		t.Errorf("mismatch (-want, +got):\n%s", diff)
+	for _, tc := range cases {
+		tc := tc
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+
+			ctx := project.TestContext(t)
+			logger := logging.FromContext(ctx)
+
+			gen := &keyGenerator{}
+
+			// batch time, one hour after midnight.
+			// Will be used as the offset when creating TEKs.
+			batchTime := timeutils.UTCMidnight(time.Now().UTC()).Add(time.Hour)
+
+			sameDay := timeutils.UTCMidnight(batchTime)
+
+			input := []*exportproto.TemporaryExposureKey{
+				{ // Valid key, fully expired, missing report type and transmission risk
+					KeyData:                    gen.fakeExposureKey(t),
+					RollingStartIntervalNumber: proto.Int32(pubmodel.IntervalNumber(sameDay.Add(-24 * time.Hour))),
+					RollingPeriod:              proto.Int32(144),
+				},
+				{ // same day key, should be future dated
+					KeyData:                    gen.fakeExposureKey(t),
+					RollingStartIntervalNumber: proto.Int32(pubmodel.IntervalNumber(sameDay)),
+					RollingPeriod:              proto.Int32(144),
+				},
+			}
+
+			appPackageName := "import-export-test"
+			regions := []string{"US"}
+			settings := &transformer{
+				appPackageName: appPackageName,
+				importRegions:  regions,
+				traveler:       tc.traveler,
+				batchTime:      batchTime,
+				truncateWindow: time.Hour,
+				exportImportID: 7,
+				importFileID:   42,
+				exportImportConfig: &pubmodel.ExportImportConfig{
+					DefaultReportType:         verifyapi.ReportTypeConfirmed,
+					BackfillSymptomOnset:      true,
+					BackfillSymptomOnsetValue: 10,
+					MaxSymptomOnsetDays:       28,
+					AllowClinical:             false,
+					AllowRevoked:              false,
+				},
+				logger: logger,
+			}
+
+			got, _ := settings.transform(input)
+
+			want := []*pubmodel.Exposure{
+				{
+					ExposureKey:           input[0].KeyData,
+					TransmissionRisk:      2,
+					AppPackageName:        appPackageName,
+					Regions:               regions,
+					Traveler:              tc.traveler,
+					IntervalNumber:        input[0].GetRollingStartIntervalNumber(),
+					IntervalCount:         input[0].GetRollingPeriod(),
+					CreatedAt:             batchTime,
+					ExportImportID:        &settings.exportImportID,
+					ImportFileID:          &settings.importFileID,
+					ReportType:            verifyapi.ReportTypeConfirmed,
+					DaysSinceSymptomOnset: &settings.exportImportConfig.BackfillSymptomOnsetValue,
+				},
+				{ // Key hasn't yet expired, so it is future dated into tomorrow.
+					ExposureKey:           input[1].KeyData,
+					TransmissionRisk:      2,
+					AppPackageName:        appPackageName,
+					Regions:               regions,
+					Traveler:              tc.traveler,
+					IntervalNumber:        input[1].GetRollingStartIntervalNumber(),
+					IntervalCount:         input[1].GetRollingPeriod(),
+					CreatedAt:             timeutils.UTCMidnight(batchTime.Add(24 * time.Hour)).Add(settings.truncateWindow),
+					ExportImportID:        &settings.exportImportID,
+					ImportFileID:          &settings.importFileID,
+					ReportType:            verifyapi.ReportTypeConfirmed,
+					DaysSinceSymptomOnset: &settings.exportImportConfig.BackfillSymptomOnsetValue,
+				},
+			}
+
+			if diff := cmp.Diff(want, got, cmpopts.IgnoreUnexported(pubmodel.Exposure{})); diff != "" {
+				t.Errorf("mismatch (-want, +got):\n%s", diff)
+			}
+		})
 	}
 }
diff --git a/internal/exportimport/model/exportimport_test.go b/internal/exportimport/model/exportimport_test.go
index 61448af..62170fb 100644
--- a/internal/exportimport/model/exportimport_test.go
+++ b/internal/exportimport/model/exportimport_test.go
@@ -69,6 +69,16 @@ func TestValidate(t *testing.T) {
 			},
 			want: "",
 		},
+		{
+			name: "valid2",
+			ei: &ExportImport{
+				Region:     "US",
+				IndexFile:  "a/index.txt",
+				ExportRoot: "a",
+				Traveler:   true,
+			},
+			want: "",
+		},
 	}
 
 	for _, tc := range cases {