Repository: deislabs/ratify
Pull request #162 (2022-04-12 17:34:46)

Use ORAS FetchReference
- Use the `FetchReference` method in ORAS to fetch blob content
- Remove `blobDesc` parameter from referrer store API as it's redundant
- Update `getRawContentFromCache` to use `ioutil.ReadAll` to avoid relying on Descriptor `size`


Patch:
diff --git a/README.md b/README.md
index 20ea41d..09115d8 100644
--- a/README.md
+++ b/README.md
@@ -26,6 +26,8 @@ by its developers, nor is it "supported" software.
 
 Try out ratify in Kuberenetes through Gatekeeper as the admission controller.
 
+Prerequisite: Kubernetes v1.20 or higher
+
 - Setup Gatekeeper with [external data](https://open-policy-agent.github.io/gatekeeper/website/docs/externaldata)
 
 ```bash
@@ -35,9 +37,11 @@ helm install gatekeeper/gatekeeper  \
     --name-template=gatekeeper \
     --namespace gatekeeper-system --create-namespace \
     --set enableExternalData=true \
-    --set controllerManager.dnsPolicy=ClusterFirst,audit.dnsPolicy=ClusterFirst
+    --set validatingWebhookTimeoutSeconds=7
 ```
 
+NOTE: `validatingWebhookTimeoutSeconds` increased from 3 to 7 so all Ratify operations complete in complex scenarios. Kubernetes v1.20 or higher is REQUIRED to increase timeout.  
+
 - Deploy ratify and a `demo` constraint on gatekeeper
 
 ```bash
diff --git a/charts/ratify-gatekeeper/values.yaml b/charts/ratify-gatekeeper/values.yaml
index 34d444d..0fcf5f0 100644
--- a/charts/ratify-gatekeeper/values.yaml
+++ b/charts/ratify-gatekeeper/values.yaml
@@ -8,7 +8,7 @@ gatekeeper:
   auditFromCache: false
   disableMutation: false
   disableValidatingWebhook: false
-  validatingWebhookTimeoutSeconds: 3
+  validatingWebhookTimeoutSeconds: 7
   validatingWebhookFailurePolicy: Ignore
   validatingWebhookCheckIgnoreFailurePolicy: Fail
   enableDeleteOperations: false
diff --git a/charts/ratify/templates/configmap.yaml b/charts/ratify/templates/configmap.yaml
index a2ee985..3521645 100644
--- a/charts/ratify/templates/configmap.yaml
+++ b/charts/ratify/templates/configmap.yaml
@@ -5,6 +5,9 @@ metadata:
 data:
   config.json: |
     {
+      "executor": {
+        "requestTimeout": 6800
+      },
       "stores": {
         "version": "1.0.0",
         "plugins": [
diff --git a/cmd/ratify/cmd/referrer.go b/cmd/ratify/cmd/referrer.go
index 4ec3f2b..3b562ab 100644
--- a/cmd/ratify/cmd/referrer.go
+++ b/cmd/ratify/cmd/referrer.go
@@ -26,7 +26,6 @@ import (
 	"github.com/deislabs/ratify/pkg/ocispecs"
 	sf "github.com/deislabs/ratify/pkg/referrerstore/factory"
 	"github.com/deislabs/ratify/pkg/utils"
-	v1 "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/spf13/cobra"
 )
 
@@ -152,7 +151,7 @@ func showBlob(opts referrerCmdOptions) error {
 
 	for _, referrerStore := range stores {
 		if referrerStore.Name() == opts.storeName {
-			content, err := referrerStore.GetBlobContent(context.Background(), subRef, digest, v1.Descriptor{})
+			content, err := referrerStore.GetBlobContent(context.Background(), subRef, digest)
 			if err != nil {
 				return err
 			}
diff --git a/cmd/ratify/cmd/serve.go b/cmd/ratify/cmd/serve.go
index d90bcd0..070b03d 100644
--- a/cmd/ratify/cmd/serve.go
+++ b/cmd/ratify/cmd/serve.go
@@ -92,6 +92,7 @@ func serve(opts serveCmdOptions) error {
 		Verifiers:      verifiers,
 		ReferrerStores: stores,
 		PolicyEnforcer: policyEnforcer,
+		Config:         &cf.ExecutorConfig,
 	}
 
 	if opts.httpServerAddress != "" {
diff --git a/docs/README.md b/docs/README.md
index f68ee49..568d017 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -74,7 +74,7 @@ type ReferrerStore interface {
  Name() string
  ListReferrers(ctx context.Context, subjectReference common.Reference, artifactTypes []string, nextToken string, subjectDesc *ocispecs.SubjectReference) (ListReferrersResult, error)
  // Used for small objects.
- GetBlobContent(ctx context.Context, subjectReference common.Reference, digest digest.Digest, blobDesc oci.Descriptor) ([]byte, error)
+ GetBlobContent(ctx context.Context, subjectReference common.Reference, digest digest.Digest) ([]byte, error)
  GetReferenceManifest(ctx context.Context, subjectReference common.Reference, referenceDesc ocispecs.ReferenceDescriptor) (ocispecs.ReferenceManifest, error)
 }
 ```
diff --git a/go.mod b/go.mod
index e5312fa..76e3ea2 100644
--- a/go.mod
+++ b/go.mod
@@ -24,5 +24,5 @@ require (
 	k8s.io/api v0.22.5
 	k8s.io/apimachinery v0.22.5
 	k8s.io/client-go v0.22.5
-	oras.land/oras-go/v2 v2.0.0-20220329045430-dd7d79fb99a1
+	oras.land/oras-go/v2 v2.0.0-20220411024522-db4a48eade0e
 )
diff --git a/go.sum b/go.sum
index 0d71faf..ca28325 100644
--- a/go.sum
+++ b/go.sum
@@ -2907,8 +2907,8 @@ mvdan.cc/interfacer v0.0.0-20180901003855-c20040233aed/go.mod h1:Xkxe497xwlCKkIa
 mvdan.cc/lint v0.0.0-20170908181259-adc824a0674b/go.mod h1:2odslEg/xrtNQqCYg2/jCoyKnw3vv5biOc3JnIcYfL4=
 mvdan.cc/unparam v0.0.0-20210104141923-aac4ce9116a7/go.mod h1:hBpJkZE8H/sb+VRFvw2+rBpHNsTBcvSpk61hr8mzXZE=
 nhooyr.io/websocket v1.8.7/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0=
-oras.land/oras-go/v2 v2.0.0-20220329045430-dd7d79fb99a1 h1:MQBoC4YYIOnGtLqMGCvK+eQnfopaRmY1yoBxr/OA9uU=
-oras.land/oras-go/v2 v2.0.0-20220329045430-dd7d79fb99a1/go.mod h1:0IQiLwHUJuMs0+QYGavaeQWw5FD4ABD/RP5YamXT/sc=
+oras.land/oras-go/v2 v2.0.0-20220411024522-db4a48eade0e h1:njS1UlPEXosOodJ9T2NGMJPmgiEzAwSHDJPfnjq7nXo=
+oras.land/oras-go/v2 v2.0.0-20220411024522-db4a48eade0e/go.mod h1:0IQiLwHUJuMs0+QYGavaeQWw5FD4ABD/RP5YamXT/sc=
 pack.ag/amqp v0.11.2/go.mod h1:4/cbmt4EJXSKlG6LCfWHoqmN0uFdy5i/+YFz+fTfhV4=
 pgregory.net/rapid v0.3.3/go.mod h1:UYpPVyjFHzYBGHIxLFoupi8vwk6rXNzRY9OMvVxFIOU=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
diff --git a/pkg/referrerstore/api.go b/pkg/referrerstore/api.go
index 991dabb..ed2da7f 100644
--- a/pkg/referrerstore/api.go
+++ b/pkg/referrerstore/api.go
@@ -22,7 +22,6 @@ import (
 	"github.com/deislabs/ratify/pkg/ocispecs"
 	"github.com/deislabs/ratify/pkg/referrerstore/config"
 	"github.com/opencontainers/go-digest"
-	oci "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
 // ListReferrersResult represents the result of ListReferrers API
@@ -42,7 +41,7 @@ type ReferrerStore interface {
 
 	// GetBlobContent returns the blob with the given digest
 	// WARNING: This API is intended to use for small objects like signatures, SBoMs
-	GetBlobContent(ctx context.Context, subjectReference common.Reference, digest digest.Digest, blobDesc oci.Descriptor) ([]byte, error)
+	GetBlobContent(ctx context.Context, subjectReference common.Reference, digest digest.Digest) ([]byte, error)
 
 	// GetReferenceManifest returns the reference artifact manifest as given by the descriptor
 	GetReferenceManifest(ctx context.Context, subjectReference common.Reference, referenceDesc ocispecs.ReferenceDescriptor) (ocispecs.ReferenceManifest, error)
diff --git a/pkg/referrerstore/mocks/types.go b/pkg/referrerstore/mocks/types.go
index 55b556f..3856940 100644
--- a/pkg/referrerstore/mocks/types.go
+++ b/pkg/referrerstore/mocks/types.go
@@ -24,7 +24,6 @@ import (
 	"github.com/deislabs/ratify/pkg/referrerstore"
 	"github.com/deislabs/ratify/pkg/referrerstore/config"
 	"github.com/opencontainers/go-digest"
-	oci "github.com/opencontainers/image-spec/specs-go/v1"
 	v1 "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
@@ -41,7 +40,7 @@ func (s *TestStore) ListReferrers(ctx context.Context, subjectReference common.R
 	return referrerstore.ListReferrersResult{Referrers: s.References}, nil
 }
 
-func (s *TestStore) GetBlobContent(ctx context.Context, subjectReference common.Reference, digest digest.Digest, blobDesc oci.Descriptor) ([]byte, error) {
+func (s *TestStore) GetBlobContent(ctx context.Context, subjectReference common.Reference, digest digest.Digest) ([]byte, error) {
 	return nil, nil
 }
 
diff --git a/pkg/referrerstore/oras/oras.go b/pkg/referrerstore/oras/oras.go
index 05af50d..cec07b5 100644
--- a/pkg/referrerstore/oras/oras.go
+++ b/pkg/referrerstore/oras/oras.go
@@ -22,6 +22,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"io/ioutil"
 	"net/http"
 	paths "path/filepath"
 	"time"
@@ -169,42 +170,38 @@ func (store *orasStore) ListReferrers(ctx context.Context, subjectReference comm
 	return referrerstore.ListReferrersResult{Referrers: referrers}, nil
 }
 
-func (store *orasStore) GetBlobContent(ctx context.Context, subjectReference common.Reference, digest digest.Digest, blobDesc oci.Descriptor) ([]byte, error) {
+func (store *orasStore) GetBlobContent(ctx context.Context, subjectReference common.Reference, digest digest.Digest) ([]byte, error) {
 	var err error
 	repository, expiry, err := store.createRepository(ctx, subjectReference)
 	if err != nil {
 		return nil, err
 	}
 
-	// resolve blob descriptor if not provided
-	var resolvedBlobDesc oci.Descriptor
-	if blobDesc.Digest != "" {
-		resolvedBlobDesc = blobDesc
-	} else {
-		ref := fmt.Sprintf("%s@%s", subjectReference.Path, digest)
-		resolvedBlobDesc, err = repository.Blobs().Resolve(ctx, ref)
-		if err != nil {
-			store.evictAuthCache(subjectReference.Original, err)
-			return nil, err
-		}
+	// create a dummy Descriptor to check the local store cache
+	blobDescriptor := oci.Descriptor{
+		Digest: digest,
+		Size:   0, // dummy size value
 	}
 
 	// check if blob exists in local ORAS cache
-	isCached, err := store.localCache.Exists(ctx, resolvedBlobDesc)
+	isCached, err := store.localCache.Exists(ctx, blobDescriptor)
 	if err != nil {
 		return nil, err
 	}
 
 	if !isCached {
+		// generate the reference path with digest
+		ref := fmt.Sprintf("%s@%s", subjectReference.Path, digest)
+
 		// fetch blob content from remote repository
-		rc, err := repository.Fetch(ctx, resolvedBlobDesc)
+		blobDesc, rc, err := repository.Blobs().FetchReference(ctx, ref)
 		if err != nil {
 			store.evictAuthCache(subjectReference.Original, err)
 			return nil, err
 		}
 
 		// push fetched content to local ORAS cache
-		err = store.localCache.Push(ctx, resolvedBlobDesc, rc)
+		err = store.localCache.Push(ctx, blobDesc, rc)
 		if err != nil {
 			return nil, err
 		}
@@ -213,7 +210,7 @@ func (store *orasStore) GetBlobContent(ctx context.Context, subjectReference com
 	// add the repository client to the auth cache if all repository operations successful
 	store.addAuthCache(subjectReference.Original, repository, expiry)
 
-	return store.getRawContentFromCache(ctx, resolvedBlobDesc)
+	return store.getRawContentFromCache(ctx, blobDescriptor)
 }
 
 func (store *orasStore) GetReferenceManifest(ctx context.Context, subjectReference common.Reference, referenceDesc ocispecs.ReferenceDescriptor) (ocispecs.ReferenceManifest, error) {
@@ -349,8 +346,8 @@ func (store *orasStore) getRawContentFromCache(ctx context.Context, descriptor o
 	if err != nil {
 		return nil, err
 	}
-	buf := make([]byte, descriptor.Size)
-	_, err = reader.Read(buf)
+
+	buf, err := ioutil.ReadAll(reader)
 	if err != nil {
 		return nil, err
 	}
diff --git a/pkg/referrerstore/plugin/plugin.go b/pkg/referrerstore/plugin/plugin.go
index b0797c2..5137061 100644
--- a/pkg/referrerstore/plugin/plugin.go
+++ b/pkg/referrerstore/plugin/plugin.go
@@ -29,7 +29,6 @@ import (
 	"github.com/deislabs/ratify/pkg/referrerstore/config"
 	"github.com/deislabs/ratify/pkg/referrerstore/types"
 	"github.com/opencontainers/go-digest"
-	oci "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
 // StorePlugin describes a store that is implemented by invoking the plugins
@@ -97,7 +96,7 @@ func (sp *StorePlugin) Name() string {
 	return sp.name
 }
 
-func (sp *StorePlugin) GetBlobContent(ctx context.Context, subjectReference common.Reference, digest digest.Digest, blobDesc oci.Descriptor) ([]byte, error) {
+func (sp *StorePlugin) GetBlobContent(ctx context.Context, subjectReference common.Reference, digest digest.Digest) ([]byte, error) {
 	pluginPath, err := sp.executor.FindInPaths(sp.name, sp.path)
 	if err != nil {
 		return nil, err
diff --git a/pkg/referrerstore/plugin/skel/skel.go b/pkg/referrerstore/plugin/skel/skel.go
index 8c0bc2e..3949639 100644
--- a/pkg/referrerstore/plugin/skel/skel.go
+++ b/pkg/referrerstore/plugin/skel/skel.go
@@ -32,7 +32,6 @@ import (
 	"github.com/deislabs/ratify/pkg/referrerstore/types"
 	"github.com/deislabs/ratify/pkg/utils"
 	"github.com/opencontainers/go-digest"
-	oci "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
 type pcontext struct {
@@ -43,7 +42,7 @@ type pcontext struct {
 }
 
 type ListReferrers func(args *CmdArgs, subjectReference common.Reference, artifactTypes []string, nextToken string, subjectDesc *ocispecs.SubjectDescriptor) (*referrerstore.ListReferrersResult, error)
-type GetBlobContent func(args *CmdArgs, subjectReference common.Reference, digest digest.Digest, blobDesc oci.Descriptor) ([]byte, error)
+type GetBlobContent func(args *CmdArgs, subjectReference common.Reference, digest digest.Digest) ([]byte, error)
 type GetReferenceManifest func(args *CmdArgs, subjectReference common.Reference, digest digest.Digest) (ocispecs.ReferenceManifest, error)
 type GetSubjectDescriptor func(args *CmdArgs, subjectReference common.Reference) (*ocispecs.SubjectDescriptor, error)
 
@@ -154,7 +153,7 @@ func (c *pcontext) cmdGetBlob(cmdArgs *CmdArgs, pluginFunc GetBlobContent) *plug
 		return plugin.NewError(types.ErrArgsParsingFailure, fmt.Sprintf("cannot parse digest arg %s", digestArg), err.Error())
 	}
 
-	result, err := pluginFunc(cmdArgs, cmdArgs.subjectRef, digest, oci.Descriptor{})
+	result, err := pluginFunc(cmdArgs, cmdArgs.subjectRef, digest)
 
 	if err != nil {
 		return plugin.NewError(types.ErrPluginCmdFailure, fmt.Sprintf("plugin command %s failed", sp.ListReferrersCommand), err.Error())
diff --git a/pkg/verifier/notaryv2/notaryv2.go b/pkg/verifier/notaryv2/notaryv2.go
index c65e0ca..cf5c651 100644
--- a/pkg/verifier/notaryv2/notaryv2.go
+++ b/pkg/verifier/notaryv2/notaryv2.go
@@ -117,7 +117,7 @@ func (v *notaryV2Verifier) Verify(ctx context.Context,
 	}
 
 	for _, blobDesc := range referenceManifest.Blobs {
-		refBlob, err := store.GetBlobContent(ctx, subjectReference, blobDesc.Digest, blobDesc)
+		refBlob, err := store.GetBlobContent(ctx, subjectReference, blobDesc.Digest)
 		if err != nil {
 			return verifier.VerifierResult{IsSuccess: false}, err
 		}
diff --git a/plugins/referrerstore/sample/sample.go b/plugins/referrerstore/sample/sample.go
index e4b161c..c266dc3 100644
--- a/plugins/referrerstore/sample/sample.go
+++ b/plugins/referrerstore/sample/sample.go
@@ -41,7 +41,7 @@ func ListReferrers(args *skel.CmdArgs, subjectReference common.Reference, artifa
 	}, nil
 }
 
-func GetBlobContent(args *skel.CmdArgs, subjectReference common.Reference, digest digest.Digest, blobDesc v1.Descriptor) ([]byte, error) {
+func GetBlobContent(args *skel.CmdArgs, subjectReference common.Reference, digest digest.Digest) ([]byte, error) {
 	return []byte(digest.String()), nil
 }
 
diff --git a/plugins/verifier/licensechecker/licensechecker.go b/plugins/verifier/licensechecker/licensechecker.go
index c6adcdd..c04fe14 100644
--- a/plugins/verifier/licensechecker/licensechecker.go
+++ b/plugins/verifier/licensechecker/licensechecker.go
@@ -68,7 +68,7 @@ func VerifyReference(args *skel.CmdArgs, subjectReference common.Reference, desc
 	}
 
 	for _, blobDesc := range referenceManifest.Blobs {
-		refBlob, err := store.GetBlobContent(ctx, subjectReference, blobDesc.Digest, blobDesc)
+		refBlob, err := store.GetBlobContent(ctx, subjectReference, blobDesc.Digest)
 		if err != nil {
 			return nil, err
 		}
diff --git a/plugins/verifier/sbom/sbom.go b/plugins/verifier/sbom/sbom.go
index 0812c81..0908026 100644
--- a/plugins/verifier/sbom/sbom.go
+++ b/plugins/verifier/sbom/sbom.go
@@ -76,7 +76,7 @@ func VerifyReference(args *skel.CmdArgs, subjectReference common.Reference, refe
 	}
 
 	for _, blobDesc := range referenceManifest.Blobs {
-		refBlob, err := referrerStore.GetBlobContent(ctx, subjectReference, blobDesc.Digest, blobDesc)
+		refBlob, err := referrerStore.GetBlobContent(ctx, subjectReference, blobDesc.Digest)
 		if err != nil {
 			return nil, err
 		}


Test patch:
diff --git a/pkg/referrerstore/plugin/plugin_test.go b/pkg/referrerstore/plugin/plugin_test.go
index 766a2fd..b0a72d6 100644
--- a/pkg/referrerstore/plugin/plugin_test.go
+++ b/pkg/referrerstore/plugin/plugin_test.go
@@ -26,7 +26,6 @@ import (
 	"github.com/deislabs/ratify/pkg/ocispecs"
 	"github.com/deislabs/ratify/pkg/referrerstore/config"
 	"github.com/opencontainers/go-digest"
-	v1 "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
 type TestExecutor struct {
@@ -111,7 +110,7 @@ func TestPluginMain_GetBlobContent_InvokeExpected(t *testing.T) {
 	subject := common.Reference{
 		Original: "localhost",
 	}
-	result, err := storePlugin.GetBlobContent(context.Background(), subject, "", v1.Descriptor{})
+	result, err := storePlugin.GetBlobContent(context.Background(), subject, "")
 	if err != nil {
 		t.Fatalf("plugin execution failed %v", err)
 	}
diff --git a/pkg/referrerstore/plugin/skel/skel_test.go b/pkg/referrerstore/plugin/skel/skel_test.go
index 8c30623..7348227 100644
--- a/pkg/referrerstore/plugin/skel/skel_test.go
+++ b/pkg/referrerstore/plugin/skel/skel_test.go
@@ -31,7 +31,7 @@ import (
 )
 
 func TestPluginMain_GetBlobContent_ReturnsExpected(t *testing.T) {
-	getBlobContent := func(args *CmdArgs, subjectReference common.Reference, digest digest.Digest, blobDesc v1.Descriptor) ([]byte, error) {
+	getBlobContent := func(args *CmdArgs, subjectReference common.Reference, digest digest.Digest) ([]byte, error) {
 		return []byte(digest.String()), nil
 	}
 	environment := map[string]string{
@@ -171,7 +171,7 @@ func TestPluginMain_GetSubjectDesc_ReturnsExpected(t *testing.T) {
 }
 
 func TestPluginMain_ErrorCases(t *testing.T) {
-	getBlobContent := func(args *CmdArgs, subjectReference common.Reference, digest digest.Digest, blobDesc v1.Descriptor) ([]byte, error) {
+	getBlobContent := func(args *CmdArgs, subjectReference common.Reference, digest digest.Digest) ([]byte, error) {
 		return nil, fmt.Errorf("simulated error")
 	}
 	environment := map[string]string{
@@ -244,7 +244,7 @@ func TestPluginMain_ErrorCases(t *testing.T) {
 }
 
 func TestPluginMain_GetBlobContent_ErrorCases(t *testing.T) {
-	getBlobContent := func(args *CmdArgs, subjectReference common.Reference, digest digest.Digest, blobDesc v1.Descriptor) ([]byte, error) {
+	getBlobContent := func(args *CmdArgs, subjectReference common.Reference, digest digest.Digest) ([]byte, error) {
 		return []byte(digest.String()), nil
 	}
 	environment := map[string]string{