Repository: deislabs/ratify
Pull request #215 (2022-06-09 22:58:57)

Update executor on configuration file update
# Description

This PR includes the first set of changes to allow for dynamic configuration. This change adds a file watcher on the ratify configuration file and reload the all components of executor when needed. Please see design doc [here](https://hackmd.io/7sFQLTnZQfm7XHXH1X7IZw)

- adding file watcher to configuration file and reload executor on change detected
- moved createExecutor from configuration logic from serve.go into 
- Adding FileHash to configuration to detect if file has changed
Fixes # (9)

## Type of change

- [ ] New feature (non-breaking change which adds functionality)
- [ ] This change requires a documentation update
Will document an example of configMap update after merging
# How Has This Been Tested?

Please describe the tests that you ran to verify your changes. Please also list any relevant details for your test configuration
 
- Validate executor component(Policy/Verifier/Store) reloaded correctly in a local server and cluster environment
1. Validate an image, expected invalid result
2. Update the config and run verify again and got a valid result

update scenario tested:
helm upgrade ratify ./charts/ratify --atomic
kubectl replace -f ratifyconfigmap.yaml

 

susanshi: This PR is ready, @akashsinghal @mnltejaswini , please review when you get a chance.
susanshi: @mnltejaswini ,  thanks for the review, much appreciated! I have updated the implementation to address the feedback, please review when you have a chance.
dtzar: @susanshi - looks like you have some merge conflicts
susanshi: merge conflict resolved, thanks.
mnltejaswini: @susanshi @dtzar I have few minor comments, otherwise, LGTM. Thank you

Patch:
diff --git a/.github/workflows/build-pr.yml b/.github/workflows/build-pr.yml
index c5beda0..17bb219 100644
--- a/.github/workflows/build-pr.yml
+++ b/.github/workflows/build-pr.yml
@@ -23,7 +23,7 @@ jobs:
         run: bin/ratify version
 
   build_test_e2e:
-    name: "[External Data] Build and run e2e Test"
+    name: "Build and run e2e Test"
     runs-on: ubuntu-latest
     timeout-minutes: 15
     permissions:
diff --git a/Makefile b/Makefile
index a61ef1e..b9e54b7 100644
--- a/Makefile
+++ b/Makefile
@@ -115,7 +115,8 @@ e2e-deploy-gatekeeper: e2e-helm-install
     --name-template=gatekeeper \
     --namespace gatekeeper-system --create-namespace \
     --set enableExternalData=true \
-    --set validatingWebhookTimeoutSeconds=7
+    --set validatingWebhookTimeoutSeconds=7 \
+    --set auditInterval=0
 
 e2e-deploy-ratify:
 	docker build -f ./httpserver/Dockerfile -t localbuild:test . 
diff --git a/cmd/ratify/cmd/serve.go b/cmd/ratify/cmd/serve.go
index 5621492..751fa41 100644
--- a/cmd/ratify/cmd/serve.go
+++ b/cmd/ratify/cmd/serve.go
@@ -20,10 +20,6 @@ import (
 
 	"github.com/deislabs/ratify/config"
 	"github.com/deislabs/ratify/httpserver"
-	ef "github.com/deislabs/ratify/pkg/executor/core"
-	pf "github.com/deislabs/ratify/pkg/policyprovider/factory"
-	sf "github.com/deislabs/ratify/pkg/referrerstore/factory"
-	vf "github.com/deislabs/ratify/pkg/verifier/factory"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 )
@@ -59,44 +55,14 @@ func NewCmdServe(argv ...string) *cobra.Command {
 }
 
 func serve(opts serveCmdOptions) error {
-	cf, err := config.Load(opts.configFilePath)
-	if err != nil {
-		return err
-	}
-
-	logrus.Info("configuration successfully loaded.")
-	stores, err := sf.CreateStoresFromConfig(cf.StoresConfig, config.GetDefaultPluginPath())
-
-	if err != nil {
-		return err
-	}
-	logrus.Infof("stores successfully created. number of stores %d", len(stores))
-
-	verifiers, err := vf.CreateVerifiersFromConfig(cf.VerifiersConfig, config.GetDefaultPluginPath())
 
+	getExecutor, err := config.GetExecutorAndWatchForUpdate(opts.configFilePath)
 	if err != nil {
 		return err
 	}
 
-	logrus.Infof("verifiers successfully created. number of verifiers %d", len(verifiers))
-
-	policyEnforcer, err := pf.CreatePolicyProviderFromConfig(cf.PoliciesConfig)
-
-	if err != nil {
-		return err
-	}
-
-	logrus.Infof("policies successfully created.")
-
-	executor := ef.Executor{
-		Verifiers:      verifiers,
-		ReferrerStores: stores,
-		PolicyEnforcer: policyEnforcer,
-		Config:         &cf.ExecutorConfig,
-	}
-
 	if opts.httpServerAddress != "" {
-		server, err := httpserver.NewServer(context.Background(), opts.httpServerAddress, &executor)
+		server, err := httpserver.NewServer(context.Background(), opts.httpServerAddress, getExecutor)
 		if err != nil {
 			return err
 		}
diff --git a/config/config.go b/config/config.go
index 51765cf..cf2c3a6 100644
--- a/config/config.go
+++ b/config/config.go
@@ -16,10 +16,11 @@ limitations under the License.
 package config
 
 import (
+	"crypto/sha256"
+	"encoding/hex"
 	"encoding/json"
-	"errors"
 	"fmt"
-	"io"
+	"io/ioutil"
 	"os"
 	"path/filepath"
 	"sync"
@@ -42,6 +43,7 @@ type Config struct {
 	PoliciesConfig  pcConfig.PoliciesConfig  `json:"policy,omitempty"`
 	VerifiersConfig vfConfig.VerifiersConfig `json:"verifier,omitempty"`
 	ExecutorConfig  exConfig.ExecutorConfig  `json:"executor,omitempty"`
+	fileHash        string                   `json:"-"`
 }
 
 var (
@@ -75,26 +77,21 @@ func getHomeDir() string {
 func Load(configFilePath string) (Config, error) {
 
 	config := Config{}
-	if configFilePath == "" {
 
-		if configDir == "" {
-			initConfigDir.Do(InitDefaultPaths)
-		}
+	body, readErr := ioutil.ReadFile(configFilePath)
 
-		configFilePath = defaultConfigFilePath
+	if readErr != nil {
+		return config, fmt.Errorf("unable to read config file at path %s", readErr)
 	}
 
-	file, err := os.OpenFile(configFilePath, os.O_RDONLY, 0644)
+	err := json.Unmarshal(body, &config)
 	if err != nil {
-		if os.IsNotExist(err) {
-			return config, fmt.Errorf("could not find config file at path %s", configFilePath)
-		}
-		return config, err
+		return config, fmt.Errorf("unable to unmarshal config body: %v", err)
 	}
-	defer file.Close()
 
-	if err := json.NewDecoder(file).Decode(&config); err != nil && !errors.Is(err, io.EOF) {
-		return config, err
+	config.fileHash, err = getFileHash(body)
+	if err != nil {
+		return config, fmt.Errorf("error getting configuration file hash error %v", err)
 	}
 
 	return config, nil
@@ -106,3 +103,26 @@ func GetDefaultPluginPath() string {
 	}
 	return defaultPluginsPath
 }
+
+// if configFilePath is empty, return configuration path from environment variable
+func getConfigurationFile(configFilePath string) string {
+	if configFilePath == "" {
+		if configDir == "" {
+			initConfigDir.Do(InitDefaultPaths)
+		}
+
+		return defaultConfigFilePath
+	}
+	return configFilePath
+}
+
+func getFileHash(file []byte) (fileHash string, err error) {
+	hash := sha256.New()
+
+	length, err := hash.Write(file)
+	if err != nil || length == 0 {
+		return "", fmt.Errorf("unable to generate hash for configuration file, err '%v', hash length %v", err, length)
+	}
+
+	return hex.EncodeToString(hash.Sum(nil)), nil
+}
diff --git a/config/configManager.go b/config/configManager.go
new file mode 100644
index 0000000..4b51735
--- /dev/null
+++ b/config/configManager.go
@@ -0,0 +1,216 @@
+/*
+Copyright The Ratify Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package config
+
+import (
+	"os"
+	"time"
+
+	ef "github.com/deislabs/ratify/pkg/executor/core"
+	"github.com/deislabs/ratify/pkg/policyprovider"
+	pf "github.com/deislabs/ratify/pkg/policyprovider/factory"
+	"github.com/deislabs/ratify/pkg/referrerstore"
+	sf "github.com/deislabs/ratify/pkg/referrerstore/factory"
+	"github.com/deislabs/ratify/pkg/verifier"
+	vf "github.com/deislabs/ratify/pkg/verifier/factory"
+	"github.com/fsnotify/fsnotify"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+type GetExecutor func() *ef.Executor
+
+var (
+	configHash string
+	executor   ef.Executor
+)
+
+// Create a executor from configurationFile and setup config file watcher
+func GetExecutorAndWatchForUpdate(configFilePath string) (GetExecutor, error) {
+
+	configFilePath = getConfigurationFile(configFilePath)
+	cf, err := Load(configFilePath)
+
+	if err != nil {
+		return func() *ef.Executor { return &ef.Executor{} }, err
+	}
+
+	configHash = cf.fileHash
+
+	stores, verifiers, policyEnforcer, err := createFromConfig(cf)
+
+	if err != nil {
+		return func() *ef.Executor { return &ef.Executor{} }, err
+	}
+
+	executor = ef.Executor{
+		Verifiers:      verifiers,
+		ReferrerStores: stores,
+		PolicyEnforcer: policyEnforcer,
+		Config:         &cf.ExecutorConfig,
+	}
+
+	err = watchForConfigurationChange(configFilePath)
+
+	if err != nil {
+		return func() *ef.Executor { return &ef.Executor{} }, err
+	}
+
+	logrus.Info("configuration successfully loaded.")
+
+	return func() *ef.Executor { return &executor }, nil
+}
+
+// Returns created referer store, verifier, policyprovider objects from config
+func createFromConfig(cf Config) ([]referrerstore.ReferrerStore, []verifier.ReferenceVerifier, policyprovider.PolicyProvider, error) {
+	stores, err := sf.CreateStoresFromConfig(cf.StoresConfig, GetDefaultPluginPath())
+
+	if err != nil {
+		return nil, nil, nil, errors.Wrap(err, "failed to load store from config")
+	}
+	logrus.Infof("stores successfully created. number of stores %d", len(stores))
+
+	verifiers, err := vf.CreateVerifiersFromConfig(cf.VerifiersConfig, GetDefaultPluginPath())
+
+	if err != nil {
+		return nil, nil, nil, errors.Wrap(err, "failed to load verifiers from config")
+	}
+
+	logrus.Infof("verifiers successfully created. number of verifiers %d", len(verifiers))
+
+	policyEnforcer, err := pf.CreatePolicyProviderFromConfig(cf.PoliciesConfig)
+
+	if err != nil {
+		return nil, nil, nil, errors.Wrap(err, "failed to load policy provider from config")
+	}
+
+	logrus.Infof("policies successfully created.")
+
+	return stores, verifiers, policyEnforcer, nil
+}
+
+func reloadExecutor(configFilePath string) {
+
+	cf, err := Load(configFilePath)
+
+	if err != nil {
+		logrus.Warnf("failed to load from config file , err: %v", err)
+		return
+	}
+
+	if configHash != cf.fileHash {
+		stores, verifiers, policyEnforcer, err := createFromConfig(cf)
+
+		newExecutor := ef.Executor{
+			Verifiers:      verifiers,
+			ReferrerStores: stores,
+			PolicyEnforcer: policyEnforcer,
+			Config:         &cf.ExecutorConfig,
+		}
+
+		if err != nil {
+			logrus.Warnf("failed to store/verifier/policy objects from config, no updates loaded. err: %v", err)
+			return
+		}
+
+		executor = newExecutor
+		configHash = cf.fileHash
+		logrus.Infof("configuration file has been updated, reloading executor succeeded")
+	} else {
+		logrus.Infof("no change found in config file, no executor update needed")
+	}
+	return
+}
+
+// Setup a watcher on file at configFilePath, reload executor on file change
+func watchForConfigurationChange(configFilePath string) error {
+
+	watcher, err := fsnotify.NewWatcher()
+	if err != nil {
+		errors.Wrap(err, "new file watcher on configuration file failed ")
+	}
+
+	err = watcher.Add(configFilePath)
+
+	if err != nil {
+		logrus.Errorf("adding configuration file watcher failed, err: %v", err)
+		return err
+	}
+
+	logrus.Infof("watcher added on configuration file %v", configFilePath)
+
+	// setup for loop to listen for events
+	go func() {
+
+		for {
+			select {
+			case event, ok := <-watcher.Events:
+				if !ok {
+					logrus.Warnf("no longer watching configuration file changes, file watcher event channel closed")
+					return
+				}
+
+				logrus.Infof("file watcher event detected %v", event)
+
+				// In a cluster scenario, a configMap will recreate the config file
+				// after the remove event, the watcher will also be removed
+				// since a watcher on a non existent file is not supported, we sleep until the file exist add the watcher back
+				if event.Name == configFilePath && event.Op&fsnotify.Remove == fsnotify.Remove {
+					logrus.Infof("config file remove event detected")
+					sleepTime := 1 * time.Second
+					waitTime := 60 //1min
+
+					time.Sleep(sleepTime)
+					_, err := os.Stat(configFilePath)
+
+					for err != nil {
+						if waitTime < 0 {
+							logrus.Warnf("config file not found after waiting for %v sec, os.Stat error %v", waitTime, err)
+							return
+						}
+						logrus.Infof("config file does not exist yet, sleeping again")
+						_, err = os.Stat(configFilePath)
+						time.Sleep(sleepTime)
+						waitTime--
+					}
+					reloadExecutor(configFilePath)
+					err = watcher.Add(configFilePath)
+
+					if err != nil {
+						logrus.Errorf("adding configuration file watcher failed, err: %v", err)
+						continue
+					}
+
+					logrus.Infof("watcher added on configuration directory %v", configFilePath)
+
+				}
+
+				// In a local scenario, the configuration will be updated through a write event
+				if event.Name == configFilePath && event.Op&fsnotify.Write == fsnotify.Write {
+					reloadExecutor(configFilePath)
+				}
+
+			case err, ok := <-watcher.Errors:
+				if !ok {
+					logrus.Errorf("configuration file watcher returned error : %v, watcher will be closed.", err)
+					return
+				}
+			}
+		}
+	}()
+
+	return nil
+}
diff --git a/httpserver/handlers.go b/httpserver/handlers.go
index 9822db7..63a2ffc 100644
--- a/httpserver/handlers.go
+++ b/httpserver/handlers.go
@@ -55,7 +55,7 @@ func (server *Server) verify(ctx context.Context, w http.ResponseWriter, r *http
 			Subject: subject,
 		}
 
-		result, err := server.Executor.VerifySubject(ctx, verifyParameters)
+		result, err := server.GetExecutor().VerifySubject(ctx, verifyParameters)
 
 		if err != nil {
 			return err
diff --git a/httpserver/server.go b/httpserver/server.go
index 6851170..76c0aa5 100644
--- a/httpserver/server.go
+++ b/httpserver/server.go
@@ -21,7 +21,7 @@ import (
 	"net"
 	"net/http"
 
-	ef "github.com/deislabs/ratify/pkg/executor/core"
+	"github.com/deislabs/ratify/config"
 	"github.com/gorilla/mux"
 )
 
@@ -31,23 +31,23 @@ const (
 
 type (
 	Server struct {
-		Address  string
-		Router   *mux.Router
-		Executor *ef.Executor
-		Context  context.Context
+		Address     string
+		Router      *mux.Router
+		GetExecutor config.GetExecutor
+		Context     context.Context
 	}
 )
 
-func NewServer(context context.Context, address string, executor *ef.Executor) (*Server, error) {
+func NewServer(context context.Context, address string, getExecutor config.GetExecutor) (*Server, error) {
 	if address == "" {
 		return nil, ServerAddrNotFoundError{}
 	}
 
 	server := &Server{
-		Address:  address,
-		Executor: executor,
-		Router:   mux.NewRouter(),
-		Context:  context,
+		Address:     address,
+		GetExecutor: getExecutor,
+		Router:      mux.NewRouter(),
+		Context:     context,
 	}
 	server.registerHandlers()
 
@@ -78,7 +78,7 @@ func (server *Server) register(method, path string, handler ContextHandler) {
 }
 
 func (server *Server) registerHandlers() {
-	server.register("POST", ServerRootURL+"/verify", processTimeout(server.verify, server.Executor.GetVerifyRequestTimeout()))
+	server.register("POST", ServerRootURL+"/verify", processTimeout(server.verify, server.GetExecutor().GetVerifyRequestTimeout()))
 }
 
 type ServerAddrNotFoundError struct{}
diff --git a/pkg/executor/core/executor.go b/pkg/executor/core/executor.go
index 2343546..7a3b036 100644
--- a/pkg/executor/core/executor.go
+++ b/pkg/executor/core/executor.go
@@ -46,8 +46,8 @@ type Executor struct {
 
 // TODO Logging within executor
 func (executor Executor) VerifySubject(ctx context.Context, verifyParameters e.VerifyParameters) (types.VerifyResult, error) {
-	result, err := executor.verifySubjectInternal(ctx, verifyParameters)
 
+	result, err := executor.verifySubjectInternal(ctx, verifyParameters)
 	if err != nil {
 		// get the result for the error based on the policy.
 		// Do we need to consider no referrers as success or failure?
@@ -129,7 +129,9 @@ func (executor Executor) verifySubjectInternal(ctx context.Context, verifyParame
 func (ex Executor) verifyReference(ctx context.Context, subjectRef common.Reference, subjectDesc *ocispecs.SubjectDescriptor, referenceDesc ocispecs.ReferenceDescriptor, referrerStore referrerstore.ReferrerStore) types.VerifyResult {
 	var verifyResults []interface{}
 	var isSuccess = true
+
 	for _, verifier := range ex.Verifiers {
+
 		if verifier.CanVerify(ctx, referenceDesc) {
 			verifyResult, err := verifier.Verify(ctx, subjectRef, referenceDesc, referrerStore, ex)
 			verifyResult.Subject = subjectRef.String()
diff --git a/pkg/referrerstore/factory/factory.go b/pkg/referrerstore/factory/factory.go
index 35bfd3f..3a5c4bc 100644
--- a/pkg/referrerstore/factory/factory.go
+++ b/pkg/referrerstore/factory/factory.go
@@ -58,7 +58,7 @@ func CreateStoresFromConfig(storesConfig config.StoresConfig, defaultPluginPath
 	}
 
 	if len(storesConfig.Stores) == 0 {
-		return nil, errors.New("referrer store config should have atleast one store")
+		return nil, errors.New("referrer store config should have at least one store")
 	}
 
 	var stores []referrerstore.ReferrerStore


Test patch:
diff --git a/config/config_test.go b/config/config_test.go
index cf865be..d790a59 100644
--- a/config/config_test.go
+++ b/config/config_test.go
@@ -41,7 +41,8 @@ func TestLoad_FromDefaultPath(t *testing.T) {
 		t.Fatalf("config file creation failed %v", err)
 	}
 
-	config, err := Load("")
+	configurationPath := getConfigurationFile("")
+	config, err := Load(configurationPath)
 	if err != nil {
 		t.Fatalf("loading config failed %v", err)
 	}
@@ -56,7 +57,7 @@ func TestLoad_FromDefaultPath(t *testing.T) {
 	}
 }
 
-func TestLoad_FromGiventPath(t *testing.T) {
+func TestLoad_FromGivenPath(t *testing.T) {
 	tmpDir, err := ioutil.TempDir("", "test-config")
 	if err != nil {
 		t.Fatalf("temp dir creation failed %v", err)
@@ -109,7 +110,7 @@ func TestLoad_EmptyConfigSucceeds(t *testing.T) {
 	defer os.RemoveAll(tmpDir)
 
 	fileName := filepath.Join(tmpDir, ConfigFileName)
-	content := []byte("")
+	content := []byte("{}")
 	err = ioutil.WriteFile(fileName, content, 0644)
 	if err != nil {
 		t.Fatalf("config file creation failed %v", err)
@@ -145,3 +146,34 @@ func TestLoad_InvalidConfigFile(t *testing.T) {
 		t.Fatalf("loading config is expected to failed")
 	}
 }
+
+func TestLoad_ComputeHash(t *testing.T) {
+	tmpDir, err := ioutil.TempDir("", "test-config")
+	if err != nil {
+		t.Fatalf("temp dir creation failed %v", err)
+	}
+
+	defer os.RemoveAll(tmpDir)
+
+	fileName := filepath.Join(tmpDir, ConfigFileName)
+	content := []byte(`{"store":  { "version": "1.0.0" }}`)
+	err = ioutil.WriteFile(fileName, content, 0644)
+	if err != nil {
+		t.Fatalf("config file creation failed %v", err)
+	}
+
+	config, err := Load(fileName)
+	if err != nil {
+		t.Fatalf("loading config failed %v", err)
+	}
+
+	if config.StoresConfig.Version != "1.0.0" {
+		t.Fatalf("mismatch of the loaded config expected version %s actual %s", "1.0.0", config.StoresConfig.Version)
+	}
+
+	expectedHash := "97660cbbd5c340a844fd5093a7afbccb68673fa2e418cd74528078cf018b60cb"
+
+	if config.fileHash != expectedHash {
+		t.Fatalf("Unexpected configuration hash, expected %v, actual %v", expectedHash, config.fileHash)
+	}
+}
diff --git a/httpserver/server_test.go b/httpserver/server_test.go
index 84d2359..2c39df1 100644
--- a/httpserver/server_test.go
+++ b/httpserver/server_test.go
@@ -26,6 +26,8 @@ import (
 	"github.com/deislabs/ratify/pkg/executor/core"
 	"github.com/deislabs/ratify/pkg/ocispecs"
 	config "github.com/deislabs/ratify/pkg/policyprovider/configpolicy"
+	"github.com/sirupsen/logrus"
+
 	"github.com/deislabs/ratify/pkg/policyprovider/types"
 	"github.com/deislabs/ratify/pkg/referrerstore"
 	"github.com/deislabs/ratify/pkg/referrerstore/mocks"
@@ -39,8 +41,11 @@ func TestServer_Timeout_Failed(t *testing.T) {
 	testImageName := "localhost:5000/net-monitor:v1"
 	t.Run("server_timeout_fail", func(t *testing.T) {
 		body := new(bytes.Buffer)
+
 		json.NewEncoder(body).Encode(externaldata.NewProviderRequest([]string{testImageName}))
 		request := httptest.NewRequest(http.MethodPost, "/ratify/gatekeeper/v1/verify", bytes.NewReader(body.Bytes()))
+		logrus.Infof("policies successfully created. %s", body.Bytes())
+
 		responseRecorder := httptest.NewRecorder()
 
 		testDigest := digest.FromString("test")
@@ -71,14 +76,19 @@ func TestServer_Timeout_Failed(t *testing.T) {
 			ReferrerStores: []referrerstore.ReferrerStore{store},
 			Verifiers:      []verifier.ReferenceVerifier{ver},
 		}
+
+		getExecutor := func() *core.Executor {
+			return ex
+		}
+
 		server := &Server{
-			Executor: ex,
-			Context:  request.Context(),
+			GetExecutor: getExecutor,
+			Context:     request.Context(),
 		}
 
 		handler := contextHandler{
 			context: server.Context,
-			handler: processTimeout(server.verify, server.Executor.GetVerifyRequestTimeout()),
+			handler: processTimeout(server.verify, server.GetExecutor().GetVerifyRequestTimeout()),
 		}
 
 		handler.ServeHTTP(responseRecorder, request)
diff --git a/test/bats/helpers.bash b/test/bats/helpers.bash
index 14180db..aee6877 100644
--- a/test/bats/helpers.bash
+++ b/test/bats/helpers.bash
@@ -16,4 +16,20 @@ assert_failure() {
     echo "output: $output"
     return 1
   fi
-}
\ No newline at end of file
+}
+
+wait_for_process() {
+  wait_time="$1"
+  sleep_time="$2"
+  cmd="$3"
+  while [ "$wait_time" -gt 0 ]; do
+    if eval "$cmd"; then
+      return 0
+    else
+      sleep "$sleep_time"
+      wait_time=$((wait_time - sleep_time))
+    fi
+  done
+  return 1
+}
+
diff --git a/test/bats/test.bats b/test/bats/test.bats
index 08311fc..fae0465 100644
--- a/test/bats/test.bats
+++ b/test/bats/test.bats
@@ -1,6 +1,11 @@
 #!/usr/bin/env bats
 
 load helpers
+
+BATS_TESTS_DIR=${BATS_TESTS_DIR:-test/bats/tests}
+WAIT_TIME=60
+SLEEP_TIME=1
+
 @test "quick start test" {
     run kubectl apply -f ./library/default/template.yaml
     assert_success
@@ -13,3 +18,29 @@ load helpers
     run kubectl run demo1 --image=ratify.azurecr.io/testimage:unsigned
     assert_failure
 }
+
+@test "configmap update test" {
+    run kubectl apply -f ./library/default/template.yaml
+    assert_success
+    sleep 5
+    run kubectl apply -f ./library/default/samples/constraint.yaml
+    assert_success
+    sleep 5
+    run kubectl run demo2 --image=ratify.azurecr.io/testimage:signed
+    assert_success
+
+    run kubectl get configmaps ratify-configuration --namespace=ratify-service -o yaml > currentConfig.yaml
+    run kubectl delete -f ./library/default/samples/constraint.yaml
+                                            
+    wait_for_process ${WAIT_TIME} ${SLEEP_TIME} "kubectl replace --namespace=ratify-service -f ${BATS_TESTS_DIR}/configmap/invalidconfigmap.yaml"
+    echo "Waiting for 150 second for configuration update"
+    sleep 150
+
+    run kubectl apply -f ./library/default/samples/constraint.yaml
+    assert_success
+    run kubectl run demo3 --image=ratify.azurecr.io/testimage:signed
+    echo "Current time after validate : $(date +"%T")"
+    assert_failure
+     
+    wait_for_process ${WAIT_TIME} ${SLEEP_TIME} "kubectl replace --namespace=ratify-service -f currentConfig.yaml"
+}
diff --git a/test/bats/tests/configmap/invalidconfigmap.yaml b/test/bats/tests/configmap/invalidconfigmap.yaml
new file mode 100644
index 0000000..770dbf2
--- /dev/null
+++ b/test/bats/tests/configmap/invalidconfigmap.yaml
@@ -0,0 +1,40 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: ratify-configuration
+data:
+  config.json: |
+    {
+      "executor": {
+        "requestTimeout": 6800
+      },
+      "store": {
+        "version": "1.0.0",
+        "plugins": [
+            {
+                "name": "oras"
+            }
+        ]
+      },
+      "policy": {
+        "version": "1.0.0",
+        "plugin": {
+            "name": "configPolicy",
+            "artifactVerificationPolicies": {
+                "application/vnd.dev.cosign.simplesigning.v1+json": "any"
+            }
+        }
+      },
+      "verifier": {
+          "version": "1.0.0",
+          "plugins": [
+            {
+                "name":"notaryv2",
+                "artifactTypes" : "application/vnd.cncf.notary.v2.signature",
+                "verificationCerts": [
+                    "/usr/local/ratify-certs"
+                  ]
+            }
+          ]
+      }
+    }
\ No newline at end of file