Repository: rook/rook
Pull request #8583 (2021-08-24 10:01:12)

ceph: do not check ok-to-stop when OSDs are in CLBO
**Description of your changes:**

This handles the scenario where the OSDs have been created but not yet
started due to a wrong CR configuration.
For instance, when OSDs are encrypted and Vault is used to store
encryption keys, if the KV version is incorrect during the cluster
initialization the OSDs will fail to start and stay in CLBO until the
CR is updated again with the correct KV version so that it can start.

For this scenario, if the CRUSH map has no host registered yet it's fair
to assume the initialization broke and we need to fix it. So when don't
need to call ok-to-stop since it will always fail and eventually force
pass but let's not wait for nothing.

Signed-off-by: Sébastien Han <seb@redhat.com>

<!-- Please take a look at our [Contributing](https://rook.io/docs/rook/master/development-flow.html)
documentation before submitting a Pull Request!
Thank you for contributing to Rook! -->


**Which issue is resolved by this Pull Request:**
Resolves #

**Checklist:**

- [ ] **Commit Message Formatting**: Commit titles and messages follow guidelines in the [developer guide](https://rook.io/docs/rook/master/development-flow.html#commit-structure).
- [ ] **Skip Tests for Docs**: Add the flag for skipping the build if this is only a documentation change. See [here](https://github.com/rook/rook/blob/master/INSTALL.md#skip-ci) for the flag.
- [ ] **Skip Unrelated Tests**: Add a flag to run tests for a specific storage provider. See [test options](https://github.com/rook/rook/blob/master/INSTALL.md#test-storage-provider).
- [ ] Reviewed the developer guide on [Submitting a Pull Request](https://rook.io/docs/rook/master/development-flow.html#submitting-a-pull-request)
- [ ] Documentation has been updated, if necessary.
- [ ] Unit tests have been added, if necessary.
- [ ] Integration tests have been added, if necessary.
- [ ] Pending release notes updated with breaking and/or notable changes, if necessary.
- [ ] Upgrade from previous release is tested and upgrade user guide is updated, if necessary.
- [ ] Code generation (`make codegen`) has been run to update object specifications, if necessary.



Patch:
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 0a42af3c6..c55ae1524 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -57,29 +57,24 @@ jobs:
 
   linux-build-all:
     runs-on: ubuntu-18.04
-    strategy:
-      fail-fast: false
-      matrix:
-        go-version : ['1.16', '1.17']
     steps:
     - name: checkout
       uses: actions/checkout@v2
       with:
         fetch-depth: 0
 
-    - name: setup golang ${{ matrix.go-version }}
+    - name: setup golang
       uses: actions/setup-go@v2
       with:
-        go-version: ${{ matrix.go-version }}
+        go-version: 1.16
 
     - name: set up QEMU
       uses: docker/setup-qemu-action@master
       with:
         platforms: all
 
-    - name: build.all rook with go ${{ matrix.go-version }}
-      run: |
-        tests/scripts/github-action-helper.sh build_rook_all
+    - name: build.all rook
+      run: tests/scripts/github-action-helper.sh build_rook_all
 
     - name: setup tmate session for debugging
       if: failure()
diff --git a/build/crds/crds.go b/build/crds/crds.go
index 01bee7092..1bcaee88b 100644
--- a/build/crds/crds.go
+++ b/build/crds/crds.go
@@ -1,4 +1,3 @@
-//go:build crds
 // +build crds
 
 /*
diff --git a/build/makelib/golang.mk b/build/makelib/golang.mk
index 6f9b12f53..38296d70c 100644
--- a/build/makelib/golang.mk
+++ b/build/makelib/golang.mk
@@ -48,7 +48,7 @@ GO_TEST_FLAGS ?=
 # ====================================================================================
 # Setup go environment
 
-GO_SUPPORTED_VERSIONS ?= 1.16|1.17
+GO_SUPPORTED_VERSIONS ?= 1.16
 
 GO_PACKAGES := $(foreach t,$(GO_SUBDIRS),$(GO_PROJECT)/$(t)/...)
 GO_INTEGRATION_TEST_PACKAGES := $(foreach t,$(GO_INTEGRATION_TESTS_SUBDIRS),$(GO_PROJECT)/$(t)/integration)
@@ -155,11 +155,8 @@ go.vet:
 	@CGO_ENABLED=0 $(GOHOST) vet $(GO_COMMON_FLAGS) $(GO_PACKAGES) $(GO_INTEGRATION_TEST_PACKAGES)
 
 .PHONY: go.fmt
-# ignore deepcopy generated files since the tool hardcoded the header with a "// +build" which in Golang 1.17 makes it fail gofmt since "////go:build" is preferred
-# see: https://github.com/kubernetes/gengo/blob/master/examples/deepcopy-gen/generators/deepcopy.go#L136
-# https://github.com/kubernetes/gengo/pull/210
 go.fmt: $(GOFMT)
-	@gofmt_out=$$(find $(GO_SUBDIRS) -name "*.go" -not -name "*.deepcopy.go" | xargs $(GOFMT) -s -d -e 2>&1) && [ -z "$${gofmt_out}" ] || (echo "$${gofmt_out}" 1>&2; exit 1)
+	@gofmt_out=$$($(GOFMT) -s -d -e $(GO_SUBDIRS) $(GO_INTEGRATION_TESTS_SUBDIRS) 2>&1) && [ -z "$${gofmt_out}" ] || (echo "$${gofmt_out}" 1>&2; exit 1)
 
 go.validate: go.vet go.fmt
 
diff --git a/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml
similarity index 100%
rename from pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml
rename to cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml
diff --git a/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-svc.yaml b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-svc.yaml
similarity index 100%
rename from pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-svc.yaml
rename to cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin-svc.yaml
diff --git a/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin.yaml b/cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin.yaml
similarity index 100%
rename from pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin.yaml
rename to cluster/examples/kubernetes/ceph/csi/template/cephfs/csi-cephfsplugin.yaml
diff --git a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml
similarity index 100%
rename from pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml
rename to cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml
diff --git a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-svc.yaml b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-svc.yaml
similarity index 100%
rename from pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-svc.yaml
rename to cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin-svc.yaml
diff --git a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin.yaml b/cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin.yaml
similarity index 100%
rename from pkg/operator/ceph/csi/template/rbd/csi-rbdplugin.yaml
rename to cluster/examples/kubernetes/ceph/csi/template/rbd/csi-rbdplugin.yaml
diff --git a/cmd/rook/ceph/operator.go b/cmd/rook/ceph/operator.go
index 9ff69f5fa..e17db78f4 100644
--- a/cmd/rook/ceph/operator.go
+++ b/cmd/rook/ceph/operator.go
@@ -22,6 +22,7 @@ import (
 	"github.com/rook/rook/pkg/daemon/ceph/agent/flexvolume/attachment"
 	operator "github.com/rook/rook/pkg/operator/ceph"
 	cluster "github.com/rook/rook/pkg/operator/ceph/cluster"
+	"github.com/rook/rook/pkg/operator/ceph/csi"
 
 	opcontroller "github.com/rook/rook/pkg/operator/ceph/controller"
 	"github.com/rook/rook/pkg/operator/k8sutil"
@@ -41,6 +42,14 @@ https://github.com/rook/rook`,
 }
 
 func init() {
+	// csi deployment templates
+	operatorCmd.Flags().StringVar(&csi.RBDPluginTemplatePath, "csi-rbd-plugin-template-path", csi.DefaultRBDPluginTemplatePath, "path to ceph-csi rbd plugin template")
+
+	operatorCmd.Flags().StringVar(&csi.RBDProvisionerDepTemplatePath, "csi-rbd-provisioner-dep-template-path", csi.DefaultRBDProvisionerDepTemplatePath, "path to ceph-csi rbd provisioner deployment template")
+
+	operatorCmd.Flags().StringVar(&csi.CephFSPluginTemplatePath, "csi-cephfs-plugin-template-path", csi.DefaultCephFSPluginTemplatePath, "path to ceph-csi cephfs plugin template")
+	operatorCmd.Flags().StringVar(&csi.CephFSProvisionerDepTemplatePath, "csi-cephfs-provisioner-dep-template-path", csi.DefaultCephFSProvisionerDepTemplatePath, "path to ceph-csi cephfs provisioner deployment template")
+
 	operatorCmd.Flags().BoolVar(&cluster.EnableMachineDisruptionBudget, "enable-machine-disruption-budget", false, "enable fencing controllers")
 
 	flags.SetFlagsFromEnv(operatorCmd.Flags(), rook.RookEnvVarPrefix)
diff --git a/go.mod b/go.mod
index 65881bb00..94b76473f 100644
--- a/go.mod
+++ b/go.mod
@@ -15,7 +15,7 @@ require (
 	github.com/google/uuid v1.1.2
 	github.com/hashicorp/vault/api v1.0.5-0.20200902155336-f9d5ce5a171a
 	github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.1.0
-	github.com/kube-object-storage/lib-bucket-provisioner v0.0.0-20210818162813-3eee31c01875
+	github.com/kube-object-storage/lib-bucket-provisioner v0.0.0-20210311161930-4bea5edaff58
 	github.com/libopenstorage/secrets v0.0.0-20210709082113-dde442ea20ec
 	github.com/openshift/cluster-api v0.0.0-20191129101638-b09907ac6668
 	github.com/openshift/machine-api-operator v0.2.1-0.20190903202259-474e14e4965a
diff --git a/go.sum b/go.sum
index fef8d50ba..5acd43305 100644
--- a/go.sum
+++ b/go.sum
@@ -719,8 +719,8 @@ github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/kube-object-storage/lib-bucket-provisioner v0.0.0-20210818162813-3eee31c01875 h1:jX3VXgmNOye8XYKjwcTVXcBYcPv3jj657fwX8DN/HiM=
-github.com/kube-object-storage/lib-bucket-provisioner v0.0.0-20210818162813-3eee31c01875/go.mod h1:XpQ9HGG9uF5aJCBP+s6w5kSiyTIVSqCV8+XAE4qms5E=
+github.com/kube-object-storage/lib-bucket-provisioner v0.0.0-20210311161930-4bea5edaff58 h1:O9m6tfyhjr3F3yKGsgGMp1+seFyTRovmGMmbcNp6GSo=
+github.com/kube-object-storage/lib-bucket-provisioner v0.0.0-20210311161930-4bea5edaff58/go.mod h1:XpQ9HGG9uF5aJCBP+s6w5kSiyTIVSqCV8+XAE4qms5E=
 github.com/kubernetes-csi/csi-lib-utils v0.9.1 h1:sGq6ifVujfMSkfTsMZip44Ttv8SDXvsBlFk9GdYl/b8=
 github.com/kubernetes-csi/csi-lib-utils v0.9.1/go.mod h1:8E2jVUX9j3QgspwHXa6LwyN7IHQDjW9jX3kwoWnSC+M=
 github.com/kubernetes-csi/external-snapshotter/client/v4 v4.0.0/go.mod h1:YBCo4DoEeDndqvAn6eeu0vWM7QdXmHEeI9cFWplmBys=
diff --git a/images/ceph/Dockerfile b/images/ceph/Dockerfile
index 53f26151d..5e66b9bbb 100644
--- a/images/ceph/Dockerfile
+++ b/images/ceph/Dockerfile
@@ -23,6 +23,7 @@ RUN curl --fail -sSL -o /tini https://github.com/krallin/tini/releases/download/
     chmod +x /tini
 
 COPY rook rookflex toolbox.sh set-ceph-debug-level /usr/local/bin/
+COPY ceph-csi /etc/ceph-csi
 COPY ceph-monitoring /etc/ceph-monitoring
 COPY rook-external /etc/rook-external/
 COPY ceph-csv-templates /etc/ceph-csv-templates
diff --git a/images/ceph/Makefile b/images/ceph/Makefile
index eea0bb3f0..267fa68e8 100755
--- a/images/ceph/Makefile
+++ b/images/ceph/Makefile
@@ -60,6 +60,7 @@ do.build:
 	@cp set-ceph-debug-level $(TEMP)
 	@cp $(OUTPUT_DIR)/bin/linux_$(GOARCH)/rook $(TEMP)
 	@cp $(OUTPUT_DIR)/bin/linux_$(GOARCH)/rookflex $(TEMP)
+	@cp -r ../../cluster/examples/kubernetes/ceph/csi/template $(TEMP)/ceph-csi
 	@cp -r ../../cluster/examples/kubernetes/ceph/monitoring $(TEMP)/ceph-monitoring
 	@mkdir -p $(TEMP)/rook-external/test-data
 	@cp ../../cluster/examples/kubernetes/ceph/create-external-cluster-resources.* $(TEMP)/rook-external/
diff --git a/pkg/daemon/ceph/client/upgrade.go b/pkg/daemon/ceph/client/upgrade.go
index 4dd380a48..5fee97e7b 100644
--- a/pkg/daemon/ceph/client/upgrade.go
+++ b/pkg/daemon/ceph/client/upgrade.go
@@ -37,7 +37,8 @@ const (
 var (
 	// we don't perform any checks on these daemons
 	// they don't have any "ok-to-stop" command implemented
-	daemonNoCheck = []string{"mgr", "rgw", "rbd-mirror", "nfs", "fs-mirror"}
+	daemonNoCheck    = []string{"mgr", "rgw", "rbd-mirror", "nfs", "fs-mirror"}
+	errNoHostInCRUSH = errors.New("no host in crush map yet?")
 )
 
 func getCephMonVersionString(context *clusterd.Context, clusterInfo *ClusterInfo) (string, error) {
@@ -311,7 +312,7 @@ func allOSDsSameHost(context *clusterd.Context, clusterInfo *ClusterInfo) (bool,
 
 	hostOsdNodes := len(hostOsdTree.Nodes)
 	if hostOsdNodes == 0 {
-		return false, errors.New("no host in crush map yet?")
+		return false, errNoHostInCRUSH
 	}
 
 	// If the number of OSD node is 1, chances are this is simple setup with all OSDs on it
@@ -369,6 +370,10 @@ func OSDUpdateShouldCheckOkToStop(context *clusterd.Context, clusterInfo *Cluste
 	// aio means all in one
 	aio, err := allOSDsSameHost(context, clusterInfo)
 	if err != nil {
+		if errors.Is(err, errNoHostInCRUSH) {
+			logger.Warning("the CRUSH map has no 'host' entries so not performing ok-to-stop checks")
+			return false
+		}
 		logger.Warningf("failed to determine if all osds are running on the same host. will check if OSDs are ok-to-stop. if all OSDs are running on one host %s. %v", userIntervention, err)
 		return true
 	}
@@ -401,6 +406,18 @@ func osdDoNothing(context *clusterd.Context, clusterInfo *ClusterInfo) bool {
 	// aio means all in one
 	aio, err := allOSDsSameHost(context, clusterInfo)
 	if err != nil {
+		// We return true so that we can continue without a retry and subsequently not test if the
+		// osd can be stopped This handles the scenario where the OSDs have been created but not yet
+		// started due to a wrong CR configuration For instance, when OSDs are encrypted and Vault
+		// is used to store encryption keys, if the KV version is incorrect during the cluster
+		// initialization the OSDs will fail to start and stay in CLBO until the CR is updated again
+		// with the correct KV version so that it can start For this scenario we don't need to go
+		// through the path where the check whether the OSD can be stopped or not, so it will always
+		// fail and make us wait for nothing
+		if errors.Is(err, errNoHostInCRUSH) {
+			logger.Warning("the CRUSH map has no 'host' entries so not performing ok-to-stop checks")
+			return true
+		}
 		logger.Warningf("failed to determine if all osds are running on the same host, performing upgrade check anyways. %v", err)
 		return false
 	}
diff --git a/pkg/operator/ceph/csi/betav1csidriver.go b/pkg/operator/ceph/csi/betav1csidriver.go
index ec3996e0c..3d33ecd14 100644
--- a/pkg/operator/ceph/csi/betav1csidriver.go
+++ b/pkg/operator/ceph/csi/betav1csidriver.go
@@ -72,8 +72,6 @@ func (d beta1CsiDriver) createCSIDriverInfo(ctx context.Context, clientset kuber
 	// As FSGroupPolicy field is immutable, should be set only during create time.
 	// if the request is to change the FSGroupPolicy, we are deleting the CSIDriver object and creating it.
 	if driver.Spec.FSGroupPolicy != nil && csiDriver.Spec.FSGroupPolicy != nil && *driver.Spec.FSGroupPolicy != *csiDriver.Spec.FSGroupPolicy {
-		d.csiClient = csidrivers
-		d.csiDriver = csiDriver
 		return d.reCreateCSIDriverInfo(ctx)
 	}
 
@@ -90,16 +88,18 @@ func (d beta1CsiDriver) createCSIDriverInfo(ctx context.Context, clientset kuber
 }
 
 func (d beta1CsiDriver) reCreateCSIDriverInfo(ctx context.Context) error {
-	err := d.csiClient.Delete(ctx, d.csiDriver.Name, metav1.DeleteOptions{})
+	csiDriver := d.csiDriver
+	csiClient := d.csiClient
+	err := csiClient.Delete(ctx, csiDriver.Name, metav1.DeleteOptions{})
 	if err != nil {
-		return errors.Wrapf(err, "failed to delete CSIDriver object for driver %q", d.csiDriver.Name)
+		return errors.Wrapf(err, "failed to delete CSIDriver object for driver %q", csiDriver.Name)
 	}
-	logger.Infof("CSIDriver object deleted for driver %q", d.csiDriver.Name)
-	_, err = d.csiClient.Create(ctx, d.csiDriver, metav1.CreateOptions{})
+	logger.Infof("CSIDriver object deleted for driver %q", csiDriver.Name)
+	_, err = csiClient.Create(ctx, csiDriver, metav1.CreateOptions{})
 	if err != nil {
-		return errors.Wrapf(err, "failed to recreate CSIDriver object for driver %q", d.csiDriver.Name)
+		return errors.Wrapf(err, "failed to recreate CSIDriver object for driver %q", csiDriver.Name)
 	}
-	logger.Infof("CSIDriver object recreated for driver %q", d.csiDriver.Name)
+	logger.Infof("CSIDriver object recreated for driver %q", csiDriver.Name)
 	return nil
 }
 
@@ -108,7 +108,7 @@ func (d beta1CsiDriver) deleteCSIDriverInfo(ctx context.Context, clientset kuber
 	err := clientset.StorageV1beta1().CSIDrivers().Delete(ctx, name, metav1.DeleteOptions{})
 	if err != nil {
 		if apierrors.IsNotFound(err) {
-			logger.Debugf("%q CSIDriver not found; skipping deletion.", name)
+			logger.Debug("%q CSIDriver not found; skipping deletion.", name)
 			return nil
 		}
 	}
diff --git a/pkg/operator/ceph/csi/csidriver.go b/pkg/operator/ceph/csi/csidriver.go
index ec373e872..f30267462 100644
--- a/pkg/operator/ceph/csi/csidriver.go
+++ b/pkg/operator/ceph/csi/csidriver.go
@@ -72,8 +72,6 @@ func (d v1CsiDriver) createCSIDriverInfo(ctx context.Context, clientset kubernet
 	// As FSGroupPolicy field is immutable, should be set only during create time.
 	// if the request is to change the FSGroupPolicy, we are deleting the CSIDriver object and creating it.
 	if driver.Spec.FSGroupPolicy != nil && csiDriver.Spec.FSGroupPolicy != nil && *driver.Spec.FSGroupPolicy != *csiDriver.Spec.FSGroupPolicy {
-		d.csiClient = csidrivers
-		d.csiDriver = csiDriver
 		return d.reCreateCSIDriverInfo(ctx)
 	}
 
@@ -90,16 +88,18 @@ func (d v1CsiDriver) createCSIDriverInfo(ctx context.Context, clientset kubernet
 }
 
 func (d v1CsiDriver) reCreateCSIDriverInfo(ctx context.Context) error {
-	err := d.csiClient.Delete(ctx, d.csiDriver.Name, metav1.DeleteOptions{})
+	csiDriver := d.csiDriver
+	csiClient := d.csiClient
+	err := csiClient.Delete(ctx, csiDriver.Name, metav1.DeleteOptions{})
 	if err != nil {
-		return errors.Wrapf(err, "failed to delete CSIDriver object for driver %q", d.csiDriver.Name)
+		return errors.Wrapf(err, "failed to delete CSIDriver object for driver %q", csiDriver.Name)
 	}
-	logger.Infof("CSIDriver object deleted for driver %q", d.csiDriver.Name)
-	_, err = d.csiClient.Create(ctx, d.csiDriver, metav1.CreateOptions{})
+	logger.Infof("CSIDriver object deleted for driver %q", csiDriver.Name)
+	_, err = csiClient.Create(ctx, d.csiDriver, metav1.CreateOptions{})
 	if err != nil {
-		return errors.Wrapf(err, "failed to recreate CSIDriver object for driver %q", d.csiDriver.Name)
+		return errors.Wrapf(err, "failed to recreate CSIDriver object for driver %q", csiDriver.Name)
 	}
-	logger.Infof("CSIDriver object recreated for driver %q", d.csiDriver.Name)
+	logger.Infof("CSIDriver object recreated for driver %q", csiDriver.Name)
 	return nil
 }
 
@@ -108,7 +108,7 @@ func (d v1CsiDriver) deleteCSIDriverInfo(ctx context.Context, clientset kubernet
 	err := clientset.StorageV1().CSIDrivers().Delete(ctx, name, metav1.DeleteOptions{})
 	if err != nil {
 		if apierrors.IsNotFound(err) {
-			logger.Debugf("%s CSIDriver not found; skipping deletion.", name)
+			logger.Debug("%s CSIDriver not found; skipping deletion.", name)
 			return nil
 		}
 	}
diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go
index b798c8807..bc3a7bc51 100644
--- a/pkg/operator/ceph/csi/spec.go
+++ b/pkg/operator/ceph/csi/spec.go
@@ -18,7 +18,6 @@ package csi
 
 import (
 	"context"
-	_ "embed"
 	"fmt"
 	"strconv"
 	"strings"
@@ -88,6 +87,13 @@ var (
 	CephFSDriverName string
 	RBDDriverName    string
 
+	// template paths
+	RBDPluginTemplatePath         string
+	RBDProvisionerDepTemplatePath string
+
+	CephFSPluginTemplatePath         string
+	CephFSProvisionerDepTemplatePath string
+
 	// configuration map for csi
 	ConfigName = "rook-ceph-csi-config"
 	ConfigKey  = "csi-cluster-config-json"
@@ -109,22 +115,6 @@ var (
 	DefaultSnapshotterImage       = "k8s.gcr.io/sig-storage/csi-snapshotter:v4.1.1"
 	DefaultResizerImage           = "k8s.gcr.io/sig-storage/csi-resizer:v1.2.0"
 	DefaultVolumeReplicationImage = "quay.io/csiaddons/volumereplication-operator:v0.1.0"
-
-	// Local package template path for RBD
-	//go:embed template/rbd/csi-rbdplugin.yaml
-	RBDPluginTemplatePath string
-	//go:embed template/rbd/csi-rbdplugin-provisioner-dep.yaml
-	RBDProvisionerDepTemplatePath string
-	//go:embed template/rbd/csi-rbdplugin-svc.yaml
-	RBDPluginServiceTemplatePath string
-
-	// Local package template path for CephFS
-	//go:embed template/cephfs/csi-cephfsplugin.yaml
-	CephFSPluginTemplatePath string
-	//go:embed template/cephfs/csi-cephfsplugin-provisioner-dep.yaml
-	CephFSProvisionerDepTemplatePath string
-	//go:embed template/cephfs/csi-cephfsplugin-svc.yaml
-	CephFSPluginServiceTemplatePath string
 )
 
 const (
@@ -164,6 +154,15 @@ const (
 	// kubelet directory path
 	DefaultKubeletDirPath = "/var/lib/kubelet"
 
+	// template
+	DefaultRBDPluginTemplatePath         = "/etc/ceph-csi/rbd/csi-rbdplugin.yaml"
+	DefaultRBDProvisionerDepTemplatePath = "/etc/ceph-csi/rbd/csi-rbdplugin-provisioner-dep.yaml"
+	DefaultRBDPluginServiceTemplatePath  = "/etc/ceph-csi/rbd/csi-rbdplugin-svc.yaml"
+
+	DefaultCephFSPluginTemplatePath         = "/etc/ceph-csi/cephfs/csi-cephfsplugin.yaml"
+	DefaultCephFSProvisionerDepTemplatePath = "/etc/ceph-csi/cephfs/csi-cephfsplugin-provisioner-dep.yaml"
+	DefaultCephFSPluginServiceTemplatePath  = "/etc/ceph-csi/cephfs/csi-cephfsplugin-svc.yaml"
+
 	// grpc metrics and liveness port for cephfs  and rbd
 	DefaultCephFSGRPCMerticsPort     uint16 = 9091
 	DefaultCephFSLivenessMerticsPort uint16 = 9081
@@ -192,6 +191,7 @@ func CSIEnabled() bool {
 }
 
 func validateCSIParam() error {
+
 	if len(CSIParam.CSIPluginImage) == 0 {
 		return errors.New("missing csi rbd plugin image")
 	}
@@ -205,6 +205,23 @@ func validateCSIParam() error {
 		return errors.New("missing csi attacher image")
 	}
 
+	if EnableRBD {
+		if len(RBDPluginTemplatePath) == 0 {
+			return errors.New("missing rbd plugin template path")
+		}
+		if len(RBDProvisionerDepTemplatePath) == 0 {
+			return errors.New("missing rbd provisioner template path")
+		}
+	}
+
+	if EnableCephFS {
+		if len(CephFSPluginTemplatePath) == 0 {
+			return errors.New("missing cephfs plugin template path")
+		}
+		if len(CephFSProvisionerDepTemplatePath) == 0 {
+			return errors.New("missing ceph provisioner template path")
+		}
+	}
 	return nil
 }
 
@@ -413,11 +430,12 @@ func startDrivers(clientset kubernetes.Interface, rookclientset rookclient.Inter
 			return errors.Wrap(err, "failed to load rbd provisioner deployment template")
 		}
 
-		rbdService, err = templateToService("rbd-service", RBDPluginServiceTemplatePath, tp)
+		rbdService, err = templateToService("rbd-service", DefaultRBDPluginServiceTemplatePath, tp)
 		if err != nil {
 			return errors.Wrap(err, "failed to load rbd plugin service template")
 		}
 		rbdService.Namespace = namespace
+		logger.Info("successfully started CSI Ceph RBD")
 	}
 	if EnableCephFS {
 		cephfsPlugin, err = templateToDaemonSet("cephfsplugin", CephFSPluginTemplatePath, tp)
@@ -430,11 +448,12 @@ func startDrivers(clientset kubernetes.Interface, rookclientset rookclient.Inter
 			return errors.Wrap(err, "failed to load rbd provisioner deployment template")
 		}
 
-		cephfsService, err = templateToService("cephfs-service", CephFSPluginServiceTemplatePath, tp)
+		cephfsService, err = templateToService("cephfs-service", DefaultCephFSPluginServiceTemplatePath, tp)
 		if err != nil {
 			return errors.Wrap(err, "failed to load cephfs plugin service template")
 		}
 		cephfsService.Namespace = namespace
+		logger.Info("successfully started CSI CephFS driver")
 	}
 
 	// get common provisioner tolerations and node affinity
@@ -497,7 +516,6 @@ func startDrivers(clientset kubernetes.Interface, rookclientset rookclient.Inter
 			return errors.Wrapf(err, "failed to start rbd provisioner deployment: %+v", rbdProvisionerDeployment)
 		}
 		k8sutil.AddRookVersionLabelToDeployment(rbdProvisionerDeployment)
-		logger.Info("successfully started CSI Ceph RBD driver")
 	}
 
 	if rbdService != nil {
@@ -566,7 +584,6 @@ func startDrivers(clientset kubernetes.Interface, rookclientset rookclient.Inter
 			return errors.Wrapf(err, "failed to start cephfs provisioner deployment: %+v", cephfsProvisionerDeployment)
 		}
 		k8sutil.AddRookVersionLabelToDeployment(cephfsProvisionerDeployment)
-		logger.Info("successfully started CSI CephFS driver")
 	}
 	if cephfsService != nil {
 		err = ownerInfo.SetControllerReference(cephfsService)
diff --git a/pkg/operator/ceph/csi/util.go b/pkg/operator/ceph/csi/util.go
index a79d3fd60..4d3b2bf01 100644
--- a/pkg/operator/ceph/csi/util.go
+++ b/pkg/operator/ceph/csi/util.go
@@ -18,6 +18,8 @@ package csi
 
 import (
 	"bytes"
+	"io/ioutil"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"text/template"
@@ -32,10 +34,15 @@ import (
 	"k8s.io/client-go/kubernetes"
 )
 
-func loadTemplate(name, templateData string, p templateParam) (string, error) {
+func loadTemplate(name, templatePath string, p templateParam) (string, error) {
+	b, err := ioutil.ReadFile(filepath.Clean(templatePath))
+	if err != nil {
+		return "", err
+	}
+	data := string(b)
 	var writer bytes.Buffer
 	t := template.New(name)
-	t, err := t.Parse(templateData)
+	t, err = t.Parse(data)
 	if err != nil {
 		return "", errors.Wrapf(err, "failed to parse template %v", name)
 	}
@@ -43,9 +50,9 @@ func loadTemplate(name, templateData string, p templateParam) (string, error) {
 	return writer.String(), err
 }
 
-func templateToService(name, templateData string, p templateParam) (*corev1.Service, error) {
+func templateToService(name, templatePath string, p templateParam) (*corev1.Service, error) {
 	var svc corev1.Service
-	t, err := loadTemplate(name, templateData, p)
+	t, err := loadTemplate(name, templatePath, p)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to load service template")
 	}
@@ -57,9 +64,9 @@ func templateToService(name, templateData string, p templateParam) (*corev1.Serv
 	return &svc, nil
 }
 
-func templateToDaemonSet(name, templateData string, p templateParam) (*apps.DaemonSet, error) {
+func templateToDaemonSet(name, templatePath string, p templateParam) (*apps.DaemonSet, error) {
 	var ds apps.DaemonSet
-	t, err := loadTemplate(name, templateData, p)
+	t, err := loadTemplate(name, templatePath, p)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to load daemonset template")
 	}
@@ -71,18 +78,18 @@ func templateToDaemonSet(name, templateData string, p templateParam) (*apps.Daem
 	return &ds, nil
 }
 
-func templateToDeployment(name, templateData string, p templateParam) (*apps.Deployment, error) {
-	var dep apps.Deployment
-	t, err := loadTemplate(name, templateData, p)
+func templateToDeployment(name, templatePath string, p templateParam) (*apps.Deployment, error) {
+	var ds apps.Deployment
+	t, err := loadTemplate(name, templatePath, p)
 	if err != nil {
 		return nil, errors.Wrap(err, "failed to load deployment template")
 	}
 
-	err = yaml.Unmarshal([]byte(t), &dep)
+	err = yaml.Unmarshal([]byte(t), &ds)
 	if err != nil {
-		return nil, errors.Wrap(err, "failed to unmarshal deployment template")
+		return nil, errors.Wrapf(err, "failed to unmarshal deployment template")
 	}
-	return &dep, nil
+	return &ds, nil
 }
 
 func applyResourcesToContainers(clientset kubernetes.Interface, key string, podspec *corev1.PodSpec) {
diff --git a/pkg/operator/ceph/object/bucket/provisioner.go b/pkg/operator/ceph/object/bucket/provisioner.go
index 81eaeba4a..29bed4651 100644
--- a/pkg/operator/ceph/object/bucket/provisioner.go
+++ b/pkg/operator/ceph/object/bucket/provisioner.go
@@ -554,8 +554,8 @@ func (p *Provisioner) deleteOBCResourceLogError(bucketname string) {
 // Check for additional options mentioned in OBC and set them accordingly
 func (p Provisioner) setAdditionalSettings(options *apibkt.BucketOptions) error {
 	quotaEnabled := true
-	maxObjects := MaxObjectQuota(options.ObjectBucketClaim.Spec.AdditionalConfig)
-	maxSize := MaxSizeQuota(options.ObjectBucketClaim.Spec.AdditionalConfig)
+	maxObjects := MaxObjectQuota(options)
+	maxSize := MaxSizeQuota(options)
 	if maxObjects == "" && maxSize == "" {
 		return nil
 	}
@@ -665,69 +665,3 @@ func (p *Provisioner) setAdminOpsAPIClient() error {
 
 	return nil
 }
-func (p Provisioner) updateAdditionalSettings(ob *bktv1alpha1.ObjectBucket) error {
-	var maxObjectsInt64 int64
-	var maxSizeInt64 int64
-	var err error
-	var quotaEnabled bool
-	maxObjects := MaxObjectQuota(ob.Spec.Endpoint.AdditionalConfigData)
-	maxSize := MaxSizeQuota(ob.Spec.Endpoint.AdditionalConfigData)
-	if maxObjects != "" {
-		maxObjectsInt, err := strconv.Atoi(maxObjects)
-		if err != nil {
-			return errors.Wrap(err, "failed to convert maxObjects to integer")
-		}
-		maxObjectsInt64 = int64(maxObjectsInt)
-	}
-	if maxSize != "" {
-		maxSizeInt64, err = maxSizeToInt64(maxSize)
-		if err != nil {
-			return errors.Wrapf(err, "failed to parse maxSize quota for user %q", p.cephUserName)
-		}
-	}
-	objectUser, err := p.adminOpsClient.GetUser(context.TODO(), admin.User{ID: ob.Spec.Connection.AdditionalState[cephUser]})
-	if err != nil {
-		return errors.Wrapf(err, "failed to fetch user %q", p.cephUserName)
-	}
-	if *objectUser.UserQuota.Enabled &&
-		(maxObjects == "" || maxObjectsInt64 < 0) &&
-		(maxSize == "" || maxSizeInt64 < 0) {
-		quotaEnabled = false
-		err = p.adminOpsClient.SetUserQuota(context.TODO(), admin.QuotaSpec{UID: p.cephUserName, Enabled: &quotaEnabled})
-		if err != nil {
-			return errors.Wrapf(err, "failed to disable quota to user %q", p.cephUserName)
-		}
-		return nil
-	}
-
-	quotaEnabled = true
-	quotaSpec := admin.QuotaSpec{UID: p.cephUserName, Enabled: &quotaEnabled}
-
-	//MaxObject is modified
-	if maxObjects != "" && (maxObjectsInt64 != *objectUser.UserQuota.MaxObjects) {
-		quotaSpec.MaxObjects = &maxObjectsInt64
-	}
-
-	//MaxSize is modified
-	if maxSize != "" && (maxSizeInt64 != *objectUser.UserQuota.MaxSize) {
-		quotaSpec.MaxSize = &maxSizeInt64
-	}
-	err = p.adminOpsClient.SetUserQuota(context.TODO(), quotaSpec)
-	if err != nil {
-		return errors.Wrapf(err, "failed to update quota to user %q", p.cephUserName)
-	}
-
-	return nil
-}
-
-// Update is sent when only there is modification to AdditionalConfig field in OBC
-func (p Provisioner) Update(ob *bktv1alpha1.ObjectBucket) error {
-	logger.Debugf("Update event for OB: %+v", ob)
-
-	err := p.initializeDeleteOrRevoke(ob)
-	if err != nil {
-		return err
-	}
-
-	return p.updateAdditionalSettings(ob)
-}
diff --git a/pkg/operator/ceph/object/bucket/util.go b/pkg/operator/ceph/object/bucket/util.go
index 9fd4ed6ef..71faa16cf 100644
--- a/pkg/operator/ceph/object/bucket/util.go
+++ b/pkg/operator/ceph/object/bucket/util.go
@@ -23,6 +23,7 @@ import (
 	"github.com/coreos/pkg/capnslog"
 	bktv1alpha1 "github.com/kube-object-storage/lib-bucket-provisioner/pkg/apis/objectbucket.io/v1alpha1"
 	"github.com/kube-object-storage/lib-bucket-provisioner/pkg/provisioner"
+	apibkt "github.com/kube-object-storage/lib-bucket-provisioner/pkg/provisioner/api"
 	"github.com/pkg/errors"
 	storagev1 "k8s.io/api/storage/v1"
 	kerrors "k8s.io/apimachinery/pkg/api/errors"
@@ -116,10 +117,10 @@ func randomString(n int) string {
 	return string(b)
 }
 
-func MaxObjectQuota(AdditionalConfig map[string]string) string {
-	return AdditionalConfig["maxObjects"]
+func MaxObjectQuota(options *apibkt.BucketOptions) string {
+	return options.ObjectBucketClaim.Spec.AdditionalConfig["maxObjects"]
 }
 
-func MaxSizeQuota(AdditionalConfig map[string]string) string {
-	return AdditionalConfig["maxSize"]
+func MaxSizeQuota(options *apibkt.BucketOptions) string {
+	return options.ObjectBucketClaim.Spec.AdditionalConfig["maxSize"]
 }


Test patch:
diff --git a/pkg/daemon/ceph/client/upgrade_test.go b/pkg/daemon/ceph/client/upgrade_test.go
index aceefadd3..2035d84bb 100644
--- a/pkg/daemon/ceph/client/upgrade_test.go
+++ b/pkg/daemon/ceph/client/upgrade_test.go
@@ -357,4 +357,11 @@ func TestOSDUpdateShouldCheckOkToStop(t *testing.T) {
 		treeOutput = fake.OsdTreeOutput(0, 0)
 		assert.False(t, OSDUpdateShouldCheckOkToStop(context, clusterInfo))
 	})
+
+	// degraded case, OSDs are failing to start so they haven't registered in the CRUSH map yet
+	t.Run("0 nodes with down OSDs", func(t *testing.T) {
+		lsOutput = fake.OsdLsOutput(3)
+		treeOutput = fake.OsdTreeOutput(0, 1)
+		assert.False(t, OSDUpdateShouldCheckOkToStop(context, clusterInfo))
+	})
 }
diff --git a/pkg/operator/ceph/csi/spec_test.go b/pkg/operator/ceph/csi/spec_test.go
index fbd035750..272fcd417 100644
--- a/pkg/operator/ceph/csi/spec_test.go
+++ b/pkg/operator/ceph/csi/spec_test.go
@@ -27,6 +27,11 @@ import (
 )
 
 func TestStartCSI(t *testing.T) {
+	RBDPluginTemplatePath = "csi-rbdplugin.yaml"
+	RBDProvisionerDepTemplatePath = "csi-rbdplugin-provisioner-dep.yaml"
+	CephFSPluginTemplatePath = "csi-cephfsplugin.yaml"
+	CephFSProvisionerDepTemplatePath = "csi-cephfsplugin-provisioner-dep.yaml"
+
 	CSIParam = Param{
 		CSIPluginImage:   "image",
 		RegistrarImage:   "image",
diff --git a/pkg/operator/ceph/csi/util_test.go b/pkg/operator/ceph/csi/util_test.go
index 840307e75..d211afd97 100644
--- a/pkg/operator/ceph/csi/util_test.go
+++ b/pkg/operator/ceph/csi/util_test.go
@@ -17,6 +17,7 @@ limitations under the License.
 package csi
 
 import (
+	"io/ioutil"
 	"os"
 	"testing"
 
@@ -24,21 +25,97 @@ import (
 	"k8s.io/client-go/kubernetes/fake"
 )
 
+var (
+	testDSTemplate = []byte(`
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  name: test-label
+  namespace: {{ .Namespace }}
+spec:
+  selector:
+    matchLabels:
+      app: test-label
+  template:
+    metadata:
+      labels:
+        app: test-label
+    spec:
+      serviceAccount: test-sa
+      containers:
+        - name: registrar
+          image: {{ .RegistrarImage }}
+        - name: rbdplugin
+          image: {{ .CSIPluginImage }}
+        - name: cephfsplugin
+          image: {{ .CSIPluginImage }}
+`)
+	testDepTemplate = []byte(`
+kind: Deployment
+apiVersion: apps/v1
+metadata:
+  name: test-label
+  namespace: {{ .Namespace }}
+spec:
+  replicas: {{ .ProvisionerReplicas }}
+  selector:
+    matchLabels:
+      app: test-label
+  template:
+    metadata:
+      labels:
+        app: test-label
+    spec:
+      serviceAccount: test-sa
+      containers:
+        - name: csi-attacher
+          image: {{ .AttacherImage }}
+        - name: csi-snapshotter
+          image: {{ .SnapshotterImage }}
+        - name: csi-resizer
+          image: {{ .ResizerImage }}
+        - name: csi-provisioner
+          image: {{ .ProvisionerImage }}
+        - name: csi-cephfsplugin
+          image: {{ .CSIPluginImage }}
+`)
+)
+
 func TestDaemonSetTemplate(t *testing.T) {
+	tmp, err := ioutil.TempFile("", "yaml")
+	assert.Nil(t, err)
+
+	defer os.Remove(tmp.Name())
+
+	_, err = tmp.Write(testDSTemplate)
+	assert.Nil(t, err)
+	err = tmp.Close()
+	assert.Nil(t, err)
+
 	tp := templateParam{
 		Param:     CSIParam,
 		Namespace: "foo",
 	}
-	_, err := templateToDaemonSet("test-ds", RBDPluginTemplatePath, tp)
+	_, err = templateToDaemonSet("test-ds", tmp.Name(), tp)
 	assert.Nil(t, err)
 }
 
 func TestDeploymentTemplate(t *testing.T) {
+	tmp, err := ioutil.TempFile("", "yaml")
+	assert.Nil(t, err)
+
+	defer os.Remove(tmp.Name())
+
+	_, err = tmp.Write(testDepTemplate)
+	assert.Nil(t, err)
+	err = tmp.Close()
+	assert.Nil(t, err)
+
 	tp := templateParam{
 		Param:     CSIParam,
 		Namespace: "foo",
 	}
-	_, err := templateToDeployment("test-dep", RBDProvisionerDepTemplatePath, tp)
+	_, err = templateToDeployment("test-dep", tmp.Name(), tp)
 	assert.Nil(t, err)
 }
 
diff --git a/tests/framework/clients/bucket.go b/tests/framework/clients/bucket.go
index 8f3edeb86..70e73f8b5 100644
--- a/tests/framework/clients/bucket.go
+++ b/tests/framework/clients/bucket.go
@@ -53,10 +53,6 @@ func (b *BucketOperation) DeleteObc(obcName string, storageClassName string, buc
 	return b.k8sh.ResourceOperation("delete", b.manifests.GetOBC(obcName, storageClassName, bucketName, maxObject, createBucket))
 }
 
-func (b *BucketOperation) UpdateObc(obcName string, storageClassName string, bucketName string, maxObject string, createBucket bool) error {
-	return b.k8sh.ResourceOperation("apply", b.manifests.GetOBC(obcName, storageClassName, bucketName, maxObject, createBucket))
-}
-
 // CheckOBC, returns true if the obc, secret and configmap are all in the "check" state,
 // and returns false if any of these resources are not in the "check" state.
 // Check state values:
@@ -127,10 +123,3 @@ func (b *BucketOperation) GetSecretKey(obcName string) (string, error) {
 	return string(decode), nil
 
 }
-
-// Checks whether MaxObject is updated for ob
-func (b *BucketOperation) CheckOBMaxObject(obcName, maxobject string) bool {
-	obName, _ := b.k8sh.GetResource("obc", obcName, "--output", "jsonpath={.spec.objectBucketName}")
-	fetchMaxObject, _ := b.k8sh.GetResource("ob", obName, "--output", "jsonpath={.spec.endpoint.additionalConfig.maxObjects}")
-	return maxobject == fetchMaxObject
-}
diff --git a/tests/integration/ceph_base_object_test.go b/tests/integration/ceph_base_object_test.go
index cb54a0165..6cc912f90 100644
--- a/tests/integration/ceph_base_object_test.go
+++ b/tests/integration/ceph_base_object_test.go
@@ -48,12 +48,10 @@ var (
 	ObjectKey1             = "rookObj1"
 	ObjectKey2             = "rookObj2"
 	ObjectKey3             = "rookObj3"
-	ObjectKey4             = "rookObj4"
 	contentType            = "plain/text"
 	obcName                = "smoke-delete-bucket"
 	region                 = "us-east-1"
 	maxObject              = "2"
-	newMaxObject           = "3"
 	bucketStorageClassName = "rook-smoke-delete-bucket"
 )
 
@@ -282,27 +280,11 @@ func testObjectStoreOperations(s suite.Suite, helper *clients.TestClient, k8sh *
 			assert.Error(s.T(), poErr)
 		})
 
-		t.Run("test update quota on OBC bucket", func(t *testing.T) {
-			poErr := helper.BucketClient.UpdateObc(obcName, bucketStorageClassName, bucketname, newMaxObject, true)
-			assert.Nil(s.T(), poErr)
-			updated := utils.Retry(5, 2*time.Second, "OBC is updated", func() bool {
-				return helper.BucketClient.CheckOBMaxObject(obcName, newMaxObject)
-			})
-			assert.True(s.T(), updated)
-			logger.Infof("Testing the updated object limit")
-			_, poErr = s3client.PutObjectInBucket(bucketname, ObjBody, ObjectKey3, contentType)
-			assert.NoError(s.T(), poErr)
-			_, poErr = s3client.PutObjectInBucket(bucketname, ObjBody, ObjectKey4, contentType)
-			assert.Error(s.T(), poErr)
-		})
-
 		t.Run("delete objects on OBC bucket", func(t *testing.T) {
 			_, delobjErr := s3client.DeleteObjectInBucket(bucketname, ObjectKey1)
 			assert.Nil(s.T(), delobjErr)
 			_, delobjErr = s3client.DeleteObjectInBucket(bucketname, ObjectKey2)
 			assert.Nil(s.T(), delobjErr)
-			_, delobjErr = s3client.DeleteObjectInBucket(bucketname, ObjectKey3)
-			assert.Nil(s.T(), delobjErr)
 			logger.Info("Objects deleted on bucket successfully")
 		})
 	})