Repository: google/exposure-notifications-server
Pull request #494 (2020-05-30 02:13:05)

don't cache the export index
For those blob storage implementations that support it, pass
in an optional bool for whether a created object should be
cacheable, and have the worker set that to false for index.txt.

google-prow-robot: [APPROVALNOTIFIER] This PR is **NOT APPROVED**

This pull-request has been approved by: *<a href="https://github.com/google/exposure-notifications-server/pull/494#" title="Author self-approved">gurayAlsac</a>*
To complete the [pull request process](https://git.k8s.io/community/contributors/guide/owners.md#the-code-review-process), please assign **crwilcox**
You can assign the PR to them by writing `/assign @crwilcox` in a comment when ready.

The full list of commands accepted by this bot can be found [here](https://go.k8s.io/bot-commands?repo=google%2Fexposure-notifications-server).

<details open>
Needs approval from an approver in each of these files:

- **[OWNERS](https://github.com/google/exposure-notifications-server/blob/master/OWNERS)**

Approvers can indicate their approval by writing `/approve` in a comment
Approvers can cancel approval by writing `/approve cancel` in a comment
</details>
<!-- META={"approvers":["crwilcox"]} -->
gurayAlsac: /assign @mikehelmick
mikehelmick: /lgtm
/approve
google-prow-robot: [APPROVALNOTIFIER] This PR is **APPROVED**

This pull-request has been approved by: *<a href="https://github.com/google/exposure-notifications-server/pull/494#" title="Author self-approved">gurayAlsac</a>*, *<a href="https://github.com/google/exposure-notifications-server/pull/494#issuecomment-636262236" title="Approved">mikehelmick</a>*

The full list of commands accepted by this bot can be found [here](https://go.k8s.io/bot-commands?repo=google%2Fexposure-notifications-server).

The pull request process is described [here](https://git.k8s.io/community/contributors/guide/owners.md#the-code-review-process)

<details >
Needs approval from an approver in each of these files:

- ~~[OWNERS](https://github.com/google/exposure-notifications-server/blob/master/OWNERS)~~ [mikehelmick]

Approvers can indicate their approval by writing `/approve` in a comment
Approvers can cancel approval by writing `/approve cancel` in a comment
</details>
<!-- META={"approvers":[]} -->

Patch:
diff --git a/internal/export/worker.go b/internal/export/worker.go
index b319c61..93b2c23 100644
--- a/internal/export/worker.go
+++ b/internal/export/worker.go
@@ -205,7 +205,7 @@ func (s *Server) createFile(ctx context.Context, cfi createFileInfo) (string, er
 	logger.Infof("Created file %v, signed with %v keys", objectName, len(signers))
 	ctx, cancel := context.WithTimeout(ctx, blobOperationTimeout)
 	defer cancel()
-	if err := s.env.Blobstore().CreateObject(ctx, cfi.exportBatch.BucketName, objectName, data); err != nil {
+	if err := s.env.Blobstore().CreateObject(ctx, cfi.exportBatch.BucketName, objectName, data, false); err != nil {
 		return "", fmt.Errorf("creating file %s in bucket %s: %w", objectName, cfi.exportBatch.BucketName, err)
 	}
 	return objectName, nil
@@ -276,7 +276,7 @@ func (s *Server) createIndex(ctx context.Context, eb *model.ExportBatch, newObje
 	indexObjectName := exportIndexFilename(eb)
 	ctx, cancel := context.WithTimeout(ctx, blobOperationTimeout)
 	defer cancel()
-	if err := s.env.Blobstore().CreateObject(ctx, eb.BucketName, indexObjectName, data); err != nil {
+	if err := s.env.Blobstore().CreateObject(ctx, eb.BucketName, indexObjectName, data, false); err != nil {
 		return "", 0, fmt.Errorf("creating file %s in bucket %s: %w", indexObjectName, eb.BucketName, err)
 	}
 	return indexObjectName, len(objects), nil
diff --git a/internal/storage/filesystem_storage.go b/internal/storage/filesystem_storage.go
index 92c1ba4..9088bab 100644
--- a/internal/storage/filesystem_storage.go
+++ b/internal/storage/filesystem_storage.go
@@ -38,7 +38,7 @@ func NewFilesystemStorage(ctx context.Context) (Blobstore, error) {
 
 // CreateObject creates a new object on the filesystem or overwrites an existing
 // one.
-func (s *FilesystemStorage) CreateObject(ctx context.Context, folder, filename string, contents []byte) error {
+func (s *FilesystemStorage) CreateObject(ctx context.Context, folder, filename string, contents []byte, cacheable bool) error {
 	pth := filepath.Join(folder, filename)
 	if err := ioutil.WriteFile(pth, contents, 0644); err != nil {
 		return fmt.Errorf("failed to create object: %w", err)
diff --git a/internal/storage/google_cloud_storage.go b/internal/storage/google_cloud_storage.go
index 9b73685..a69319d 100644
--- a/internal/storage/google_cloud_storage.go
+++ b/internal/storage/google_cloud_storage.go
@@ -43,8 +43,13 @@ func NewGoogleCloudStorage(ctx context.Context) (Blobstore, error) {
 }
 
 // CreateObject creates a new cloud storage object or overwrites an existing one.
-func (gcs *GoogleCloudStorage) CreateObject(ctx context.Context, bucket, objectName string, contents []byte) error {
+func (gcs *GoogleCloudStorage) CreateObject(ctx context.Context, bucket, objectName string, contents []byte, cacheable bool) error {
 	wc := gcs.client.Bucket(bucket).Object(objectName).NewWriter(ctx)
+	if !cacheable {
+		wc.Metadata = map[string]string{
+			"Cache-Control": "no-cache, max-age=0",
+		}
+	}
 	if _, err := wc.Write(contents); err != nil {
 		return fmt.Errorf("storage.Writer.Write: %w", err)
 	}
diff --git a/internal/storage/noop_storage.go b/internal/storage/noop_storage.go
index 75a2340..b716671 100644
--- a/internal/storage/noop_storage.go
+++ b/internal/storage/noop_storage.go
@@ -26,7 +26,7 @@ func NewNoopBlobstore(ctx context.Context) (Blobstore, error) {
 	return &NoopBlobstore{}, nil
 }
 
-func (s *NoopBlobstore) CreateObject(ctx context.Context, folder, filename string, contents []byte) error {
+func (s *NoopBlobstore) CreateObject(ctx context.Context, folder, filename string, contents []byte, cacheable bool) error {
 	return nil
 }
 
diff --git a/internal/storage/storage.go b/internal/storage/storage.go
index c95f097..05377d4 100644
--- a/internal/storage/storage.go
+++ b/internal/storage/storage.go
@@ -37,7 +37,7 @@ type Config struct {
 // Blobstore defines the minimum interface for a blob storage system.
 type Blobstore interface {
 	// CreateObject creates or overwrites an object in the storage system.
-	CreateObject(ctx context.Context, bucket, objectName string, contents []byte) error
+	CreateObject(ctx context.Context, bucket, objectName string, contents []byte, cacheable bool) error
 
 	// DeleteObject deltes an object or does nothing if the object doesn't exist.
 	DeleteObject(ctx context.Context, bucket, objectName string) error


Test patch:
diff --git a/internal/storage/filesystem_storage_test.go b/internal/storage/filesystem_storage_test.go
index c900f4e..0a9c129 100644
--- a/internal/storage/filesystem_storage_test.go
+++ b/internal/storage/filesystem_storage_test.go
@@ -63,7 +63,7 @@ func TestFilesystemStorage_CreateObject(t *testing.T) {
 				t.Fatal(err)
 			}
 
-			err = storage.CreateObject(ctx, tc.folder, tc.filepath, tc.contents)
+			err = storage.CreateObject(ctx, tc.folder, tc.filepath, tc.contents, false)
 			if (err != nil) != tc.err {
 				t.Fatal(err)
 			}