Repository: apigee/registry (go)

Issue:
Registry API: Improve handling of compressed API specs
Because API specs can occasionally be large enough to exceed gRPC message size limits but also compress extremely well, we encourage users to compress specs before uploading them. But gzip encoding can include timestamps and other variations that make compressions non-unique, so the **SHA hashes that we compute for specs should be based on uncompressed specs** - allowing them to be unique, so that we can use them to detect meaningful changes to specs. **Spec content size should probably also be the uncompressed spec size**. 

Currently, for user convenience, the /contents singleton returns uncompressed specs, but this breaks downloads for specs greater than the gRPC maximum message size (4MB). To address this, **the GetSpecContents method should provide some mechanism for user control of the compression of returned values** - having /contents return the uncompressed specs makes it easier to import specs into third-party tools, so this should at least be available as an option, if not the default.


Pull request:
Recognize accept-encoding headers and return gzip-encoded spec contents.
With this change, if clients send "accept-encoding" headers that include "gzip", then gzipped specs will be returned *if the specs were uploaded and stored gzipped and have +gzip in their mime types*. "gzip" does not cause specs that were stored uncompressed to be returned compressed, because it might not be appropriate to compress some types (e.g. "+zip") and we don't plan to maintain an internal list of such uncompressible mime types.

This change also modifies the `registry` tool to send `accept-encoding=[gzip]` from the `registry get --contents` command handler.

This would resolve #200

Patch:
diff --git a/cmd/registry/core/get.go b/cmd/registry/core/get.go
index 0866a7f..c30240d 100644
--- a/cmd/registry/core/get.go
+++ b/cmd/registry/core/get.go
@@ -20,6 +20,7 @@ import (
 	"github.com/apigee/registry/gapic"
 	"github.com/apigee/registry/rpc"
 	"github.com/apigee/registry/server/registry/names"
+	"google.golang.org/grpc/metadata"
 )
 
 func GetProject(ctx context.Context,
@@ -105,6 +106,7 @@ func GetSpec(ctx context.Context,
 		return err
 	}
 	if getContents {
+		ctx := metadata.AppendToOutgoingContext(ctx, "accept-encoding", "gzip")
 		contents, err := client.GetApiSpecContents(ctx, &rpc.GetApiSpecContentsRequest{
 			Name: spec.GetName(),
 		})
@@ -134,6 +136,7 @@ func GetSpecRevision(ctx context.Context,
 		request := &rpc.GetApiSpecContentsRequest{
 			Name: spec.GetName(),
 		}
+		ctx := metadata.AppendToOutgoingContext(ctx, "accept-encoding", "gzip")
 		contents, err := client.GetApiSpecContents(ctx, request)
 		if err != nil {
 			return err
diff --git a/server/registry/actions_specs.go b/server/registry/actions_specs.go
index 7d97f59..6de69d1 100644
--- a/server/registry/actions_specs.go
+++ b/server/registry/actions_specs.go
@@ -25,6 +25,7 @@ import (
 	"github.com/apigee/registry/server/registry/names"
 	"google.golang.org/genproto/googleapis/api/httpbody"
 	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/status"
 	"google.golang.org/protobuf/types/known/emptypb"
 	"google.golang.org/protobuf/types/known/fieldmaskpb"
@@ -193,7 +194,8 @@ func (s *RegistryServer) GetApiSpecContents(ctx context.Context, req *rpc.GetApi
 	if err != nil {
 		return nil, err
 	}
-	if strings.Contains(spec.MimeType, "+gzip") {
+
+	if strings.Contains(spec.MimeType, "+gzip") && !incomingContextAllowsGZIP(ctx) {
 		contents, err := models.GUnzippedBytes(blob.Contents)
 		if err != nil {
 			return nil, status.Errorf(codes.FailedPrecondition, "failed to unzip contents with gzip MIME type: %s", err)
@@ -319,3 +321,16 @@ func (s *RegistryServer) UpdateApiSpec(ctx context.Context, req *rpc.UpdateApiSp
 	s.notify(ctx, rpc.Notification_UPDATED, spec.RevisionName())
 	return message, nil
 }
+
+func incomingContextAllowsGZIP(ctx context.Context) bool {
+	md, ok := metadata.FromIncomingContext(ctx)
+	if !ok {
+		return false
+	}
+	for _, a := range md["accept-encoding"] {
+		if a == "gzip" {
+			return true
+		}
+	}
+	return false
+}


Test patch:
diff --git a/server/registry/actions_specs_test.go b/server/registry/actions_specs_test.go
index 5b43cb5..1aec060 100644
--- a/server/registry/actions_specs_test.go
+++ b/server/registry/actions_specs_test.go
@@ -15,6 +15,8 @@
 package registry
 
 import (
+	"bytes"
+	"compress/gzip"
 	"context"
 	"crypto/sha256"
 	"fmt"
@@ -25,6 +27,7 @@ import (
 	"github.com/google/go-cmp/cmp"
 	"github.com/google/go-cmp/cmp/cmpopts"
 	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/metadata"
 	"google.golang.org/grpc/status"
 	"google.golang.org/protobuf/testing/protocmp"
 	"google.golang.org/protobuf/types/known/fieldmaskpb"
@@ -477,6 +480,126 @@ func TestGetApiSpecContents(t *testing.T) {
 	}
 }
 
+func gZippedBytes(input []byte) ([]byte, error) {
+	var buf bytes.Buffer
+	zw, err := gzip.NewWriterLevel(&buf, gzip.BestCompression)
+	if err != nil {
+		return nil, err
+	}
+	_, err = zw.Write(input)
+	if err != nil {
+		return nil, err
+	}
+	if err := zw.Close(); err != nil {
+		return nil, err
+	}
+	return buf.Bytes(), nil
+}
+
+func TestGetApiSpecContentsData(t *testing.T) {
+	specContentsCompressed, err := gZippedBytes(specContents)
+	if err != nil {
+		t.Fatalf("Setup/Seeding: Failed to gzip sample spec %s", err)
+	}
+	tests := []struct {
+		desc     string
+		seed     *rpc.ApiSpec
+		req      *rpc.GetApiSpecContentsRequest
+		accept   string
+		wantCode codes.Code
+		wantData []byte
+	}{
+		{
+			desc: "uncompressed mimetype with no accept-encoding header",
+			seed: &rpc.ApiSpec{
+				Name:     "projects/my-project/locations/global/apis/my-api/versions/v1/specs/my-spec",
+				MimeType: "application/x.openapi;version=3.0.0",
+				Contents: specContents,
+			},
+			req: &rpc.GetApiSpecContentsRequest{
+				Name: "projects/my-project/locations/global/apis/my-api/versions/v1/specs/my-spec",
+			},
+			wantCode: codes.OK,
+			wantData: specContents, // uncompressed upload, uncompressed response
+		},
+		{
+			desc: "uncompressed mimetype with accept-encoding=gzip",
+			seed: &rpc.ApiSpec{
+				Name:     "projects/my-project/locations/global/apis/my-api/versions/v1/specs/my-spec",
+				MimeType: "application/x.openapi;version=3.0.0",
+				Contents: specContents,
+			},
+			req: &rpc.GetApiSpecContentsRequest{
+				Name: "projects/my-project/locations/global/apis/my-api/versions/v1/specs/my-spec",
+			},
+			accept:   "gzip",
+			wantCode: codes.OK,
+			wantData: specContents, // we only return compressed data when it was uploaded by the client
+		},
+		{
+			desc: "gzip mimetype with no accept-encoding header",
+			seed: &rpc.ApiSpec{
+				Name:     "projects/my-project/locations/global/apis/my-api/versions/v1/specs/my-spec",
+				MimeType: "application/x.openapi+gzip;version=3.0.0",
+				Contents: specContentsCompressed,
+			},
+			req: &rpc.GetApiSpecContentsRequest{
+				Name: "projects/my-project/locations/global/apis/my-api/versions/v1/specs/my-spec",
+			},
+			wantCode: codes.OK,
+			wantData: specContents, // compressed upload, uncompressed response
+		},
+		{
+			desc: "gzip mimetype with accept-encoding=gzip",
+			seed: &rpc.ApiSpec{
+				Name:     "projects/my-project/locations/global/apis/my-api/versions/v1/specs/my-spec",
+				MimeType: "application/x.openapi+gzip;version=3.0.0",
+				Contents: specContentsCompressed,
+			},
+			req: &rpc.GetApiSpecContentsRequest{
+				Name: "projects/my-project/locations/global/apis/my-api/versions/v1/specs/my-spec",
+			},
+			accept:   "gzip",
+			wantCode: codes.OK,
+			wantData: specContentsCompressed, // compressed upload, compressed response
+		},
+		{
+			desc: "gzip mimetype with accept-encoding=some-unknown-type",
+			seed: &rpc.ApiSpec{
+				Name:     "projects/my-project/locations/global/apis/my-api/versions/v1/specs/my-spec",
+				MimeType: "application/x.openapi+gzip;version=3.0.0",
+				Contents: specContentsCompressed,
+			},
+			req: &rpc.GetApiSpecContentsRequest{
+				Name: "projects/my-project/locations/global/apis/my-api/versions/v1/specs/my-spec",
+			},
+			accept:   "some-unknown-type",
+			wantCode: codes.OK,
+			wantData: specContents, // uncompressed response because we don't support the requested encoding
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.desc, func(t *testing.T) {
+			ctx := context.Background()
+			server := defaultTestServer(t)
+			if err := seeder.SeedSpecs(ctx, server, test.seed); err != nil {
+				t.Fatalf("Setup/Seeding: Failed to seed registry: %s", err)
+			}
+			if test.accept != "" {
+				ctx = metadata.NewIncomingContext(ctx, metadata.Pairs("accept-encoding", test.accept))
+			}
+			resp, err := server.GetApiSpecContents(ctx, test.req)
+			if status.Code(err) != test.wantCode {
+				t.Errorf("GetApiSpecContents(%+v) returned status code %q, want %q: %v", test.req, status.Code(err), test.wantCode, err)
+			}
+			if bytes.Compare(resp.Data, test.wantData) != 0 {
+				t.Errorf("GetApiSpecContents(%+v) returned contents %v, want %v: %v", test.req, resp.Data, test.wantData, err)
+			}
+		})
+	}
+}
+
 func TestListApiSpecs(t *testing.T) {
 	tests := []struct {
 		desc      string