Repository: deislabs/ratify
Pull request #526 (2023-01-03 13:43:54)

feat: add cache to oras store
# Description

## What this PR does / why we need it:

To avoid lots of same requests to remote registry at the same time, we'd like to add a cache layer to the store API. Currently Ratify only implements the Oras store as built-in store, so we add Oras cache first. As the original implementation already supports `GetBlobContent` and `GetReferenceManifest` with cache, we only need to add cache to `ListReferrers` API.

This PR just introduced a new cache layer for Oras Store, which will invoke the original Oras API to retrieve data under the hood.

The cache package I used is [Ristretto](https://github.com/dgraph-io/ristretto). I'm open to other packages as well.
I've listed a few of cache libraries and aspects we would consider. Would appreciate suggestions on the library selection.

|  | GC optimization | Capacity | Entry size | Need explicit Close | Save to DataStore | License | TTL support|
| --- | --- |--- | --- |--- | --- |--- | --- |
| ristretto | sharded | total cost limit | different cost for each entry | yes | no | Apache 2.0 | yes |
| [bigcache ](https://github.com/allegro/bigcache)| Omit entries in the map and use []byte to put serialized entry | total number limit | no | yes | no | Apache 2.0 | unify ttl |
| [freecache](https://github.com/coocood/freecache) | Shard data into 256 segments with 2 pointer in each segment | total size limit (need to set GC percent manually) | no | no | yes | MIT | support |
| [groupcache ](https://github.com/golang/groupcache)| sharded | total size limit | no | no | no | Apache 2.0 | no |
| [go-cache](https://github.com/patrickmn/go-cache) | no sharding | unlimited | no | support by using SetFinalizer | yes | MIT | support |

## Which issue(s) this PR fixes *(optional, using `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when the PR gets merged)*:

This PR will address part of the issue: https://github.com/deislabs/ratify/issues/201

## Type of change

Please delete options that are not relevant.

- [ ] New feature (non-breaking change which adds functionality)
- [ ] This change requires a documentation update

# How Has This Been Tested?

1. Added unit tests.
2. Run CMD and K8s Add-on manually.

# Checklist:

- [ ] Does the affected code have corresponding tests?
- [ ] Are the changes documented, not just with inline documentation, but also with conceptual documentation such as an overview of a new feature, or task-based documentation like a tutorial? Consider if this change should be announced on your project blog.
- [ ] Does this introduce breaking changes that would require an announcement or bumping the major version?
- [ ] Do all new files have appropriate license header?


sajayantony: @shizhMSFT is anyone from ORAS interested in reviewing this? 
shizhMSFT: > @shizhMSFT is anyone from ORAS interested in reviewing this?

@qweeah @Wwwsylvia @wangxiaoxuan273 Could you help reviewing?

Patch:
diff --git a/charts/ratify/templates/store.yaml b/charts/ratify/templates/store.yaml
index 28ef569..6f395e7 100644
--- a/charts/ratify/templates/store.yaml
+++ b/charts/ratify/templates/store.yaml
@@ -25,4 +25,10 @@ spec:
     {{- if .Values.oras.authProviders.awsEcrBasicEnabled }}
     authProvider:
       name: awsEcrBasic
-    {{- end }}
\ No newline at end of file
+    {{- end }}
+    {{- if .Values.oras.cache.enabled }}
+    cacheEnabled: true
+    ttl: {{ .Values.oras.cache.ttl }}
+    capacity: {{ .Values.oras.cache.capacity }}
+    keyNumber: {{ .Values.oras.cache.keyNumber }}
+    {{- end }}
diff --git a/charts/ratify/values.yaml b/charts/ratify/values.yaml
index 1af4f10..37967e8 100644
--- a/charts/ratify/values.yaml
+++ b/charts/ratify/values.yaml
@@ -76,6 +76,13 @@ oras:
       endpoint: ""
       partition: "" # defaults to aws
       region: ""
+  cache:
+    # Please tune your cache parameters to get better performance on the Oras Store performance.
+    enabled: true  # ttl-based cache may cause inconsistency between cache and data source, please disable it if strong consistency is required.
+    ttl: 10  # in seconds
+    capacity: 100 # capacity is the max size in megabytes of the full cache.
+    keyNumber: 10000 # keyNumber is the number of items you expect to keep in the cache when full. It makes a difference to the cache performance, please set it close to the item count.
+
 
 provider:
   tls:
diff --git a/go.mod b/go.mod
index f2281b0..53f108e 100644
--- a/go.mod
+++ b/go.mod
@@ -11,6 +11,8 @@ require (
 	github.com/aws/aws-sdk-go-v2/config v1.18.8
 	github.com/aws/aws-sdk-go-v2/credentials v1.13.8
 	github.com/aws/aws-sdk-go-v2/service/ecr v1.15.0
+	github.com/cespare/xxhash/v2 v2.1.2
+	github.com/dgraph-io/ristretto v0.1.1
 	github.com/docker/cli v20.10.23+incompatible
 	github.com/docker/distribution v2.8.1+incompatible
 	github.com/fsnotify/fsnotify v1.5.4
@@ -40,17 +42,10 @@ require (
 	oras.land/oras-go/v2 v2.0.0
 )
 
-require (
-	cloud.google.com/go/compute/metadata v0.2.1 // indirect
-	github.com/Azure/go-ntlmssp v0.0.0-20220621081337-cb9428e4ac1e // indirect
-	github.com/go-asn1-ber/asn1-ber v1.5.4 // indirect
-	github.com/go-ldap/ldap/v3 v3.4.4 // indirect
-	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
-)
-
 require (
 	bitbucket.org/creachadair/shell v0.0.7 // indirect
 	cloud.google.com/go/compute v1.12.1 // indirect
+	cloud.google.com/go/compute/metadata v0.2.1 // indirect
 	github.com/Azure/azure-sdk-for-go/sdk/internal v1.0.0 // indirect
 	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
 	github.com/Azure/go-autorest/autorest v0.11.28 // indirect
@@ -58,6 +53,7 @@ require (
 	github.com/Azure/go-autorest/autorest/date v0.3.0 // indirect
 	github.com/Azure/go-autorest/logger v0.2.1 // indirect
 	github.com/Azure/go-autorest/tracing v0.6.0 // indirect
+	github.com/Azure/go-ntlmssp v0.0.0-20220621081337-cb9428e4ac1e // indirect
 	github.com/OneOfOne/xxhash v1.2.8 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect
 	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.21 // indirect
@@ -75,7 +71,6 @@ require (
 	github.com/blang/semver v3.5.1+incompatible // indirect
 	github.com/cenkalti/backoff/v4 v4.1.3 // indirect
 	github.com/census-instrumentation/opencensus-proto v0.3.0 // indirect
-	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4 // indirect
 	github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490 // indirect
 	github.com/containerd/stargz-snapshotter/estargz v0.12.1 // indirect
@@ -94,7 +89,9 @@ require (
 	github.com/fullstorydev/grpcurl v1.8.7 // indirect
 	github.com/fxamacker/cbor/v2 v2.4.0 // indirect
 	github.com/ghodss/yaml v1.0.0 // indirect
+	github.com/go-asn1-ber/asn1-ber v1.5.4 // indirect
 	github.com/go-chi/chi v4.1.2+incompatible // indirect
+	github.com/go-ldap/ldap/v3 v3.4.4 // indirect
 	github.com/go-logr/logr v1.2.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-logr/zapr v1.2.0 // indirect
@@ -130,6 +127,7 @@ require (
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.2 // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/imdario/mergo v0.3.12 // indirect
 	github.com/in-toto/in-toto-golang v0.3.4-0.20220709202702-fa494aaa0add // indirect
diff --git a/go.sum b/go.sum
index 5248c21..0274b4a 100644
--- a/go.sum
+++ b/go.sum
@@ -238,7 +238,11 @@ github.com/danieljoos/wincred v1.0.2/go.mod h1:SnuYRW9lp1oJrZX/dXJqr0cPK5gYXqx3E
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8=
+github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA=
 github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
+github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y=
 github.com/dnaeon/go-vcr v1.1.0 h1:ReYa/UBrRyQdant9B4fNHGoCNKw6qh6P0fsdGmZpR7c=
 github.com/docker/cli v20.10.23+incompatible h1:qwyha/T3rXk9lfuVcn533cKFc7n/6IzL5GXVAgMVPBg=
 github.com/docker/cli v20.10.23+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
@@ -1350,6 +1354,7 @@ golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ=
 golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
diff --git a/pkg/referrerstore/oras/cache.go b/pkg/referrerstore/oras/cache.go
new file mode 100644
index 0000000..46ca702
--- /dev/null
+++ b/pkg/referrerstore/oras/cache.go
@@ -0,0 +1,149 @@
+/*
+Copyright The Ratify Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package oras
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"sync"
+	"time"
+
+	"github.com/deislabs/ratify/pkg/common"
+	"github.com/deislabs/ratify/pkg/ocispecs"
+	"github.com/deislabs/ratify/pkg/referrerstore"
+
+	"github.com/cespare/xxhash/v2"
+	"github.com/dgraph-io/ristretto"
+	"github.com/dgraph-io/ristretto/z"
+	"github.com/sirupsen/logrus"
+)
+
+var (
+	memoryCache *ristretto.Cache
+	once        sync.Once
+)
+
+// operation defines the API operations of ReferrerStore.
+type operation int
+
+const (
+	operationName operation = iota
+	operationGetConfig
+	operationListReferrers
+	operationGetBlobContent
+	operationGetReferenceManifest
+	operationGetSubjectDescriptor
+)
+
+const (
+	defaultTTL       = 10
+	defaultCapacity  = 100 * 1024 * 1024 // 100 Megabytes
+	defaultKeyNumber = 10000
+)
+
+type orasStoreWithInMemoryCache struct {
+	referrerstore.ReferrerStore
+	cacheConf *cacheConf
+}
+
+type cacheConf struct {
+	Enabled   bool `json:"cacheEnabled"`
+	TTL       int  `json:"ttl"`
+	Capacity  int  `json:"capacity"`
+	KeyNumber int  `json:"keyNumber"`
+}
+
+func keyToHash(key interface{}) (uint64, uint64) {
+	if key == nil {
+		return 0, 0
+	}
+	switch k := key.(type) {
+	case string:
+		return z.MemHashString(k), xxhash.Sum64String(k)
+	default:
+		return 0, 0
+	}
+}
+
+// createCachedStore creates a new oras store decorated with in-memory cache to cache
+// results of ListReferrers API.
+func createCachedStore(storeBase referrerstore.ReferrerStore, cacheConf *cacheConf) (referrerstore.ReferrerStore, error) {
+	var err error
+	once.Do(func() {
+		memoryCache, err = ristretto.NewCache(&ristretto.Config{
+			NumCounters: int64(cacheConf.KeyNumber) * 10,         // number of keys to track frequency. Recommended to 10x the number of total item count in the cache.
+			MaxCost:     int64(cacheConf.Capacity) * 1024 * 1024, // Max size in Megabytes.
+			BufferItems: 64,                                      // number of keys per Get buffer. 64 is recommended by the ristretto library.
+			KeyToHash:   keyToHash,
+		})
+	})
+	if err != nil {
+		logrus.Errorf("could not create cache for referrers, err: %v", err)
+		return &orasStoreWithInMemoryCache{}, err
+	}
+
+	return &orasStoreWithInMemoryCache{
+		ReferrerStore: storeBase,
+		cacheConf:     cacheConf,
+	}, nil
+}
+
+func (store *orasStoreWithInMemoryCache) ListReferrers(ctx context.Context, subjectReference common.Reference, artifactTypes []string, nextToken string, subjectDesc *ocispecs.SubjectDescriptor) (referrerstore.ListReferrersResult, error) {
+	val, found := memoryCache.Get(getCacheKey(operationListReferrers, subjectReference))
+	if found {
+		if result, ok := val.(referrerstore.ListReferrersResult); ok {
+			return result, nil
+		}
+	}
+
+	result, err := store.ReferrerStore.ListReferrers(ctx, subjectReference, artifactTypes, nextToken, subjectDesc)
+	if err == nil {
+		if added := memoryCache.SetWithTTL(getCacheKey(operationListReferrers, subjectReference), result, 1, time.Duration(store.cacheConf.TTL)*time.Second); !added {
+			logrus.WithContext(ctx).Warnf("failed to add cache with key: %+v, val: %+v", subjectReference, result)
+		}
+	}
+
+	return result, err
+}
+
+func toCacheConfig(storePluginConfig map[string]interface{}) (*cacheConf, error) {
+	bytes, err := json.Marshal(storePluginConfig)
+	if err != nil {
+		return nil, fmt.Errorf("failed marshalling store plugin config: %+v to bytes, err: %w", storePluginConfig, err)
+	}
+
+	cacheConf := &cacheConf{}
+	if err := json.Unmarshal(bytes, cacheConf); err != nil {
+		return nil, fmt.Errorf("failed unmarshalling to Oras cache config, err: %w", err)
+	}
+
+	if cacheConf.TTL == 0 {
+		cacheConf.TTL = defaultTTL
+	}
+	if cacheConf.Capacity == 0 {
+		cacheConf.Capacity = defaultCapacity
+	}
+	if cacheConf.KeyNumber == 0 {
+		cacheConf.KeyNumber = defaultKeyNumber
+	}
+
+	return cacheConf, nil
+}
+
+func getCacheKey(op operation, ref common.Reference) string {
+	return fmt.Sprintf("%d+%s@%s", op, ref.Path, ref.Digest)
+}
diff --git a/pkg/referrerstore/oras/oras.go b/pkg/referrerstore/oras/oras.go
index 48be101..b27ae0c 100644
--- a/pkg/referrerstore/oras/oras.go
+++ b/pkg/referrerstore/oras/oras.go
@@ -97,6 +97,23 @@ func init() {
 }
 
 func (s *orasStoreFactory) Create(version string, storeConfig config.StorePluginConfig) (referrerstore.ReferrerStore, error) {
+	storeBase, err := createBaseStore(version, storeConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	cacheConf, err := toCacheConfig(storeBase.GetConfig().Store)
+	if err != nil {
+		return nil, err
+	}
+	if !cacheConf.Enabled {
+		return storeBase, nil
+	}
+
+	return createCachedStore(storeBase, cacheConf)
+}
+
+func createBaseStore(version string, storeConfig config.StorePluginConfig) (*orasStore, error) {
 	conf := OrasStoreConf{}
 
 	storeConfigBytes, err := json.Marshal(storeConfig)


Test patch:
diff --git a/pkg/referrerstore/oras/cache_test.go b/pkg/referrerstore/oras/cache_test.go
new file mode 100644
index 0000000..1934ea2
--- /dev/null
+++ b/pkg/referrerstore/oras/cache_test.go
@@ -0,0 +1,193 @@
+/*
+Copyright The Ratify Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package oras
+
+import (
+	"context"
+	"reflect"
+	"testing"
+	"time"
+
+	"github.com/deislabs/ratify/pkg/common"
+	"github.com/deislabs/ratify/pkg/ocispecs"
+	"github.com/deislabs/ratify/pkg/referrerstore"
+	"github.com/deislabs/ratify/pkg/referrerstore/config"
+	"github.com/opencontainers/go-digest"
+
+	oci "github.com/opencontainers/image-spec/specs-go/v1"
+)
+
+const (
+	testName = "testName"
+)
+
+var (
+	ttl          = 10
+	base         = &mockBase{}
+	pluginConfig = map[string]interface{}{
+		"cacheEnabled": true,
+		"ttl":          ttl,
+	}
+	conf = &cacheConf{
+		Enabled:   true,
+		TTL:       ttl,
+		Capacity:  100 * 1024 * 1024,
+		KeyNumber: 10000,
+	}
+	testStoreConfig = &config.StoreConfig{}
+	testBlob        = make([]byte, 0)
+	testDigest      = digest.Digest("sha256:123456")
+	testReference   = common.Reference{
+		Path:   "testRegistry/testRepo",
+		Digest: testDigest,
+	}
+	testDesc    = &ocispecs.SubjectDescriptor{}
+	testResult1 = referrerstore.ListReferrersResult{
+		Referrers: []ocispecs.ReferenceDescriptor{
+			{
+				Descriptor: oci.Descriptor{
+					Digest: testDigest,
+				},
+			},
+		},
+	}
+	testResult2 = referrerstore.ListReferrersResult{
+		NextToken: testNextToken2,
+	}
+	testNextToken1 = "1"
+	testNextToken2 = "2"
+)
+
+type mockBase struct{}
+
+func (m *mockBase) Name() string {
+	return testName
+}
+
+func (m *mockBase) GetConfig() *config.StoreConfig {
+	return testStoreConfig
+}
+
+func (m *mockBase) ListReferrers(ctx context.Context, subjectReference common.Reference, artifactTypes []string, nextToken string, subjectDesc *ocispecs.SubjectDescriptor) (referrerstore.ListReferrersResult, error) {
+	if nextToken == testNextToken1 {
+		return testResult1, nil
+	} else if nextToken == testNextToken2 {
+		return testResult2, nil
+	}
+	return referrerstore.ListReferrersResult{}, nil
+}
+
+func (m *mockBase) GetBlobContent(ctx context.Context, subjectReference common.Reference, digest digest.Digest) ([]byte, error) {
+	return testBlob, nil
+}
+
+func (m *mockBase) GetReferenceManifest(ctx context.Context, subjectReference common.Reference, referenceDesc ocispecs.ReferenceDescriptor) (ocispecs.ReferenceManifest, error) {
+	return ocispecs.ReferenceManifest{}, nil
+}
+
+func (m *mockBase) GetSubjectDescriptor(ctx context.Context, subjectReference common.Reference) (*ocispecs.SubjectDescriptor, error) {
+	return testDesc, nil
+}
+
+func TestCreateCachedStore(t *testing.T) {
+	if _, err := createCachedStore(base, conf); err != nil {
+		t.Fatalf("expect no error, got %v", err)
+	}
+}
+
+func TestName(t *testing.T) {
+	store, _ := createCachedStore(base, conf)
+
+	name := store.Name()
+	if name != testName {
+		t.Fatalf("expect name: %s, got %s", testName, name)
+	}
+}
+
+func TestGetConfig(t *testing.T) {
+	store, _ := createCachedStore(base, conf)
+
+	conf := store.GetConfig()
+	if !reflect.DeepEqual(conf, testStoreConfig) {
+		t.Fatalf("expect config: %+v, got %+v", testStoreConfig, conf)
+	}
+}
+
+func TestGetBlobContent(t *testing.T) {
+	store, _ := createCachedStore(base, conf)
+
+	blob, err := store.GetBlobContent(context.Background(), testReference, testDigest)
+	if err != nil {
+		t.Fatalf("expect no error, got %v", err)
+	}
+	if !reflect.DeepEqual(blob, testBlob) {
+		t.Fatalf("expect blob: %v, got %v", testBlob, blob)
+	}
+}
+
+func TestGetSubjectDescriptor(t *testing.T) {
+	store, _ := createCachedStore(base, conf)
+
+	desc, err := store.GetSubjectDescriptor(context.Background(), testReference)
+	if err != nil {
+		t.Fatalf("expect no error, got %v", err)
+	}
+	if !reflect.DeepEqual(testDesc, desc) {
+		t.Fatalf("expect desc: %v, got %v", desc, testDesc)
+	}
+}
+
+func TestListReferrers_CacheHit(t *testing.T) {
+	store, _ := createCachedStore(base, conf)
+
+	result, _ := store.ListReferrers(context.Background(), testReference, []string{}, testNextToken1, nil)
+
+	time.Sleep(time.Duration(ttl-5) * time.Second)
+
+	cachedResult, err := store.ListReferrers(context.Background(), testReference, []string{}, testNextToken2, nil)
+	if err != nil {
+		t.Fatalf("err should be nil, but got %v", err)
+	}
+	if !reflect.DeepEqual(result, cachedResult) {
+		t.Fatalf("cached result: %+v is different from result: %+v", cachedResult, result)
+	}
+}
+
+func TestListReferrers_CacheMiss(t *testing.T) {
+	store, _ := createCachedStore(base, conf)
+
+	result, _ := store.ListReferrers(context.Background(), testReference, []string{}, testNextToken1, nil)
+
+	time.Sleep(time.Duration(ttl+5) * time.Second)
+
+	cachedResult, err := store.ListReferrers(context.Background(), testReference, []string{}, testNextToken2, nil)
+	if err != nil {
+		t.Fatalf("err should be nil, but got %v", err)
+	}
+	if reflect.DeepEqual(result, cachedResult) {
+		t.Fatalf("cached result: %+v should be different from result: %+v", cachedResult, result)
+	}
+}
+
+func TestToCacheConfig(t *testing.T) {
+	resultCache, err := toCacheConfig(pluginConfig)
+	if err != nil {
+		t.Fatalf("expect no error, got %v", err)
+	}
+	if !reflect.DeepEqual(conf, resultCache) {
+		t.Fatalf("expect %v, got %v", conf, resultCache)
+	}
+}