Repository: dell/csi-baremetal
Pull request #435 (2021-06-24 09:47:05)

[ISSUE-434] Automated node removal Proposal
## Purpose
### Issue #434 

Automation of node removal procedure POC

## PR checklist
- [x] Add link to the issue
- [x] Choose PR label
- [ ] New unit tests added
- [ ] Modified code has meaningful comments
- [ ] All TODOs are linked with the issues
- [ ] All comments are resolved

## Testing
_Link to custom CI build_



Patch:
diff --git a/Makefile.validation b/Makefile.validation
index 0c786f6..7707643 100644
--- a/Makefile.validation
+++ b/Makefile.validation
@@ -52,7 +52,7 @@ test-sanity:
 load-operator-image:
 	docker pull ${REGISTRY}/${PROJECT}-${OPERATOR}:${OPERATOR_VERSION}
 	docker tag ${REGISTRY}/${PROJECT}-${OPERATOR}:${OPERATOR_VERSION} ${PROJECT}-${OPERATOR}:${OPERATOR_VERSION}
-	kind load docker-image ${PROJECT}-${OPERATOR}:${OPERATOR_VERSION}
+	$(KIND) load docker-image ${PROJECT}-${OPERATOR}:${OPERATOR_VERSION}
 
 kind-pull-images: kind-pull-sidecar-images
 	docker pull ${REGISTRY}/${PROJECT}-${LOOPBACK_DRIVE_MGR}:${TAG}
@@ -93,19 +93,57 @@ kind-tag-images:
 	docker tag ${REGISTRY}/${PROJECT}-${OPERATOR}:${TAG} ${PROJECT}-${OPERATOR}:${TAG}
 
 kind-load-images:
-	kind load docker-image ${CSI_PROVISIONER}:${CSI_PROVISIONER_TAG}
-	kind load docker-image ${CSI_REGISTRAR}:${CSI_REGISTRAR_TAG}
-	kind load docker-image ${CSI_ATTACHER}:${CSI_ATTACHER_TAG}
-	kind load docker-image ${CSI_RESIZER}:${CSI_RESIZER_TAG}
-	kind load docker-image ${LIVENESS_PROBE}:${LIVENESS_PROBE_TAG}
-	kind load docker-image ${BUSYBOX}:${BUSYBOX_TAG}
-	kind load docker-image docker.io/library/nginx:1.14-alpine
-	kind load docker-image docker.io/library/centos:latest
-	kind load docker-image ${PROJECT}-${LOOPBACK_DRIVE_MGR}:${TAG}
-	kind load docker-image ${PROJECT}-${NODE}:${TAG}
-	kind load docker-image ${PROJECT}-${NODE}-kernel-5.4:${TAG}
-	kind load docker-image ${PROJECT}-${CONTROLLER}:${TAG}
-	kind load docker-image ${PROJECT}-${SCHEDULER}-${EXTENDER}:${TAG}
-	kind load docker-image ${PROJECT}-${EXTENDER_PATCHER}:${TAG}
-	kind load docker-image ${PROJECT}-${SCHEDULER}:${TAG}
-	kind load docker-image ${PROJECT}-${OPERATOR}:${TAG}
+	$(KIND) load docker-image ${CSI_PROVISIONER}:${CSI_PROVISIONER_TAG}
+	$(KIND) load docker-image ${CSI_REGISTRAR}:${CSI_REGISTRAR_TAG}
+	$(KIND) load docker-image ${CSI_ATTACHER}:${CSI_ATTACHER_TAG}
+	$(KIND) load docker-image ${CSI_RESIZER}:${CSI_RESIZER_TAG}
+	$(KIND) load docker-image ${LIVENESS_PROBE}:${LIVENESS_PROBE_TAG}
+	$(KIND) load docker-image ${BUSYBOX}:${BUSYBOX_TAG}
+	$(KIND) load docker-image docker.io/library/nginx:1.14-alpine
+	$(KIND) load docker-image docker.io/library/centos:latest
+	$(KIND) load docker-image ${PROJECT}-${LOOPBACK_DRIVE_MGR}:${TAG}
+	$(KIND) load docker-image ${PROJECT}-${NODE}:${TAG}
+	$(KIND) load docker-image ${PROJECT}-${NODE}-kernel-5.4:${TAG}
+	$(KIND) load docker-image ${PROJECT}-${CONTROLLER}:${TAG}
+	$(KIND) load docker-image ${PROJECT}-${SCHEDULER}-${EXTENDER}:${TAG}
+	$(KIND) load docker-image ${PROJECT}-${EXTENDER_PATCHER}:${TAG}
+	$(KIND) load docker-image ${PROJECT}-${SCHEDULER}:${TAG}
+	$(KIND) load docker-image ${PROJECT}-${OPERATOR}:${TAG}
+
+kind-build:
+	# Get kind sources
+	wget -O kind-src https://codeload.github.com/kubernetes-sigs/kind/tar.gz/refs/tags/v$(KIND_VER)
+	tar -xzvf kind-src
+
+	# Add "--ipc=host" docker option and build binary
+	patch -p0 < $(KIND_DIR)/kind-$(KIND_VER).patch
+	cd kind-$(KIND_VER) && make build
+
+	# Copy file
+	cp kind-$(KIND_VER)/bin/kind $(KIND_DIR)
+
+	# Make executive
+	chmod +x $(KIND)
+
+	# Check
+	$(KIND) version
+
+	# Clean workdir
+	rm -rf kind-$(KIND_VER)
+	rm -f kind-src
+
+kind-install: kind-build
+	# Delete actual
+	rm -f /usr/bin/kind
+
+	# Copy executive file
+	mv $(KIND) /usr/bin
+
+	# Check
+	kind version
+
+kind-create-cluster:
+	$(KIND) create cluster --config $(KIND_DIR)/kind.yaml --image kindest/node:v1.18.2
+
+kind-delete-cluster:
+	$(KIND) delete cluster
diff --git a/docs/README.md b/docs/README.md
index e192fbb..e5f8aa1 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -1,3 +1,5 @@
+[![PR validation](https://github.com/dell/csi-baremetal/actions/workflows/pr.yml/badge.svg)](https://github.com/dell/csi-baremetal/actions/workflows/pr.yml)
+
 Bare-metal CSI Driver
 =====================
 
diff --git a/docs/fake-attach.md b/docs/fake-attach.md
index 027f5a9..036e8cf 100644
--- a/docs/fake-attach.md
+++ b/docs/fake-attach.md
@@ -1,6 +1,6 @@
 # Proposal: Fake Attach Volume
 
-Last updated: June 11 2011
+Last updated: July 1 2021
 
 
 ## Abstract
@@ -29,19 +29,30 @@ This feature will require application Operator to put specific annotation on PVC
 
 If fake-attach volume is successfully created, CSI Volume CR will be annotated with `fake-attach:yes` and `Operational Status` will equal to `MISSING`. The pod must be restarted after `Operational Status` returns to `OPERATIVE`.
 
+
 ## Implementation
 
-When `pv.attach.kubernetes.io/ignore-if-inaccessible: yes` annotation is set CSI must ignore NodeStageVolume errors and put `fake-attach: yes` annotation on CSI Volume CR. 
+### NodeStageVolume
+When `pv.attach.kubernetes.io/ignore-if-inaccessible: yes` annotation is set CSI must:
+- ignore NodeStageVolume errors
+- put `fake-attach: yes` annotation on CSI Volume CR if error is and annotation wasn't set before
+- delete `fake-attach: yes` annotation on CSI Volume CR if error is not and annotation was set before
+
+Event FakeAttachInvolved generated when `fake-attach: yes` annotation is setting.
+
+Event FakeAttachCleared generated when `fake-attach: yes` annotation is deleting.
+
+### NodePublishVolume 
+CSI must check `fake-attach` annotation and mount tmpfs volume in read-only mode.
 
-On NodePublishVolume CSI must check `fake-attach` annotation and mount
-tmpfs volume in read-only mode.
 Command to mount tmpfs volume: `mount -t tmpfs -o size=1M,ro <volumeID> <destination folder>`
 
-CSI must generate event to notify user about this Fake-Attach Volume.
+### NodeUnpublishVolume 
+tmpfs volume must be unmounted usually.
 
-On NodeUnpublishVolume tmpfs volume must be unmounted usually.
+### NodeUnstageVolume 
+Do nothing.
 
-On NodeUnstageVolume `fake-attach` annotation must be deleted.
 
 ## Assumptions (if applicable)
 
diff --git a/docs/proposals/automated-node-removal.md b/docs/proposals/automated-node-removal.md
new file mode 100644
index 0000000..718981f
--- /dev/null
+++ b/docs/proposals/automated-node-removal.md
@@ -0,0 +1,71 @@
+# Proposal: Automation of node removal procedure
+
+Last updated: 02.07.2021
+
+## Abstract
+
+Implementation of the option to delete CSI CRs from a node, which is going to be removed. 
+
+## Background
+
+User manual actions for node removal are described in [doc](https://github.com/dell/csi-baremetal/blob/master/docs/node-removal.md).
+The goal of this proposal is automating most of the steps of that procedure.
+
+## Proposal
+
+#### User API
+
+CSI - components on CSI side including CSI Operator
+
+Operator - application controller (or other service, which watch resources)
+
+Removal procedure:
+1. Taint a node with:
+- `node.dell.com/drain=drain:NoSchedule`
+2. Wait until Operator replaced PVs and destroyed pods
+3. Delete a node from kubernetes cluster
+4. CSI deletes all corresponding CR resources automatically (including Csibmnode)
+
+After removal the node can't be restored ad in [replacement](https://github.com/dell/csi-baremetal/blob/master/docs/proposals/node-replacement.md)! 
+
+
+## Implementation
+
+Set label:
+1. CSI Operator must watch Nodes on taint changing events
+2. Label the Csibmnode with `node.dell.com/drain=drain` if the Node has taint `node.dell.com/drain=drain:NoSchedule`
+
+Unset label:
+1. CSI Operator must watch Nodes on taint changing events
+2. Check, if the Csibmnode has `node.dell.com/drain=drain` label and the Node with the same ID has no `node.dell.com/drain=drain:NoSchedule` taint
+3. Delete `node.dell.com/drain=drain` label
+
+Removal:
+1. CSI Operator must watch Nodes on deleting events
+2. Start removal procedure, if if the Csibmnode has `node.dell.com/drain=drain` label and the Node with the same ID has been deleted from kubernetes cluster
+3. Delete drives, ACs
+4. Patch and delete Volumes, Lvgs, Csibmnode 
+
+## Rationale
+
+#### Alternative approach 1
+
+Perform CSI CRs deletion in CSI Node-controller. Trigger will be deleting the corresponding `csibmnode` CR.
+
+#### Alternative approach 2
+
+Delete CRs (Drive, LVG) in `Reconcile` method in appropriate controller if node has no `uuid label` or has `node removal` label/annotation
+
+Problem - if node was removed from cluster before and csi-baremetal-node pod is unavailable, CSI resources will be stacked as currently
+
+## Compatibility
+
+This proposal is valid for CSI deployed with CSI Operator
+
+## Open issues (same as [here](https://github.com/dell/csi-baremetal/blob/master/docs/node-removal.md))
+
+ID | Name | Descriptions | Status | Comments
+---| -----| -------------| ------ | --------
+ISSUE-1 | Should we do node removal procedure in node-controller? | Node-controller reconciles Csibmnodes and nodes already | Open | 
+ISSUE-2 | Should we delete related PVs and PVCs? |  | Open  | 
+ISSUE-3 | We don't provide any procedure to remove a node without deleting it from cluster |  | Open | 
diff --git a/docs/proposals/drive-replacement.md b/docs/proposals/drive-replacement.md
index b7d2538..e88253c 100644
--- a/docs/proposals/drive-replacement.md
+++ b/docs/proposals/drive-replacement.md
@@ -64,6 +64,10 @@ Negotiation between CSI and Operator is optional and will be done though annotat
 #### User
 To trigger physical drive replacement user must put the following annotation on the corresponding Drive custom resource:
   - `driveremove.csi-baremetal/replacement: ready` - informs that drive replacement is ready
+
+*User can trigger drive releasing himself. If user annotates driveCR with `health=<SUSPECT/BAD>`, drive health will be overridden with passed value.*
+
+*DR procedure can be repeated if drive Usage is `FAILED` or `RELEASED`. User should annotate driveCR with `drive=add` and drive-controller will switch Usage value to IN_USE. After restart the annotation will be deleted.* 
 ### Detailed workflow
 * When drive health changed from `GOOD` to `SUSPECT` or `BAD` CSI will:
   - Set drive operational status to `RELEASING`
diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go
index 5ce3b68..de7f4cc 100644
--- a/pkg/controller/controller.go
+++ b/pkg/controller/controller.go
@@ -192,6 +192,7 @@ func (c *CSIControllerService) CreateVolume(ctx context.Context, req *csi.Create
 	c.reqMu.Unlock()
 
 	if err != nil {
+		ll.Errorf("Failed to create volume: %v", err)
 		return nil, err
 	}
 
diff --git a/pkg/controller/node/statemonitor.go b/pkg/controller/node/statemonitor.go
index 2b0c95e..5969f79 100644
--- a/pkg/controller/node/statemonitor.go
+++ b/pkg/controller/node/statemonitor.go
@@ -233,10 +233,11 @@ func (n *ServicesStateMonitor) getPodToNodeList() (map[string]stateComponents, e
 		}
 
 		if podNode == nil {
-			//nolint
+			// nolint
 			log.Fatalf("Unable to find node for pod %s", pod.Name)
 		}
-		nodeID := string(podNode.ObjectMeta.UID)
+		// nolint - doesn't detect Exit() in Fatalf
+		nodeID := string(podNode.GetUID())
 		stateComponentsMap[nodeID] = stateComponents{podNode, pod}
 	}
 
diff --git a/pkg/crcontrollers/drive/drivecontroller.go b/pkg/crcontrollers/drive/drivecontroller.go
index a12a495..adab0db 100644
--- a/pkg/crcontrollers/drive/drivecontroller.go
+++ b/pkg/crcontrollers/drive/drivecontroller.go
@@ -36,7 +36,13 @@ type Controller struct {
 const (
 	ignore uint8 = 0
 	update uint8 = 1
-	delete uint8 = 2
+	remove uint8 = 2
+)
+
+const (
+	// Annotations for driveCR to perform restart process
+	driveRestartReplacementAnnotationKey   = "drive"
+	driveRestartReplacementAnnotationValue = "add"
 )
 
 // NewController creates new instance of Controller structure
@@ -115,7 +121,7 @@ func (c *Controller) Reconcile(req ctrl.Request) (ctrl.Result, error) {
 			log.Errorf("Failed to update Drive %s CR", driveName)
 			return ctrl.Result{}, client.IgnoreNotFound(err)
 		}
-	case delete:
+	case remove:
 		if err := c.client.DeleteCR(ctx, drive); err != nil {
 			log.Errorf("Failed to delete Drive %s CR", driveName)
 			return ctrl.Result{}, client.IgnoreNotFound(err)
@@ -166,6 +172,11 @@ func (c *Controller) handleDriveUpdate(ctx context.Context, log *logrus.Entry, d
 		}
 
 	case apiV1.DriveUsageReleased:
+		if c.restartReplacement(drive) {
+			toUpdate = true
+			break
+		}
+
 		status, found := drive.Annotations[apiV1.DriveAnnotationReplacement]
 		if !found || status != apiV1.DriveAnnotationReplacementReady {
 			break
@@ -214,7 +225,11 @@ func (c *Controller) handleDriveUpdate(ctx context.Context, log *logrus.Entry, d
 	case apiV1.DriveUsageRemoved:
 		if drive.Spec.Status == apiV1.DriveStatusOffline {
 			// drive was removed from the system. need to clean corresponding custom resource
-			return delete, nil
+			return remove, nil
+		}
+	case apiV1.DriveUsageFailed:
+		if c.restartReplacement(drive) {
+			toUpdate = true
 		}
 	}
 
@@ -258,3 +273,15 @@ func (c *Controller) checkAllVolsRemoved(volumes []*volumecrd.Volume) bool {
 	}
 	return true
 }
+
+// restartReplacement restores drive.Usage to IN_USE if CR is annotated
+// deletes the annotation to avoid event repeating
+func (c *Controller) restartReplacement(drive *drivecrd.Drive) bool {
+	if value, ok := drive.GetAnnotations()[driveRestartReplacementAnnotationKey]; ok && value == driveRestartReplacementAnnotationValue {
+		drive.Spec.Usage = apiV1.DriveUsageInUse
+		delete(drive.Annotations, driveRestartReplacementAnnotationKey)
+		return true
+	}
+
+	return false
+}
diff --git a/pkg/eventing/eventing.go b/pkg/eventing/eventing.go
index 3b49adb..91cf8f9 100644
--- a/pkg/eventing/eventing.go
+++ b/pkg/eventing/eventing.go
@@ -44,4 +44,8 @@ const (
 	DriveSuccessfullyReplaced = "DriveSuccessfullyReplaced"
 	DriveHasData              = "DriveHasData"
 	DriveClean                = "DriveClean"
+	DriveHealthOverridden     = "DriveHealthOverridden"
+
+	FakeAttachInvolved = "FakeAttachInvolved"
+	FakeAttachCleared  = "FakeAttachCleared"
 )
diff --git a/pkg/node/node.go b/pkg/node/node.go
index 65a99d4..665c36f 100644
--- a/pkg/node/node.go
+++ b/pkg/node/node.go
@@ -46,6 +46,7 @@ import (
 	"github.com/dell/csi-baremetal/pkg/common"
 	"github.com/dell/csi-baremetal/pkg/controller"
 	csibmnodeconst "github.com/dell/csi-baremetal/pkg/crcontrollers/operator/common"
+	"github.com/dell/csi-baremetal/pkg/eventing"
 )
 
 const (
@@ -199,10 +200,10 @@ func (s *CSINodeService) NodeStageVolume(ctx context.Context, req *csi.NodeStage
 
 	targetPath := getStagingPath(ll, req.GetStagingTargetPath())
 
-	isFakeAttach := false
+	isFakeAttachNeed := false
 	ignoreErrorIfFakeAttach := func(err error) {
 		if s.isPVCNeedFakeAttach(volumeID) {
-			isFakeAttach = true
+			isFakeAttachNeed = true
 		} else {
 			newStatus = apiV1.Failed
 			resp, errToReturn = nil, status.Error(codes.Internal, fmt.Sprintf("failed to stage volume: %s", err.Error()))
@@ -221,9 +222,18 @@ func (s *CSINodeService) NodeStageVolume(ctx context.Context, req *csi.NodeStage
 		}
 	}
 
-	if isFakeAttach {
-		volumeCR.Annotations["fake-attach"] = "yes"
-		ll.Warningf("Adding fake-attach annotation to the volume with ID %s", volumeID)
+	if isFakeAttachNeed {
+		if volumeCR.Annotations[fakeAttachVolumeAnnotation] != fakeAttachVolumeKey {
+			volumeCR.Annotations[fakeAttachVolumeAnnotation] = fakeAttachVolumeKey
+			ll.Warningf("Adding fake-attach annotation to the volume with ID %s", volumeID)
+			s.VolumeManager.recorder.Eventf(volumeCR, eventing.ErrorType, eventing.FakeAttachInvolved,
+				"Fake-attach involved for volume with ID %s", volumeID)
+		}
+	} else if volumeCR.Annotations[fakeAttachVolumeAnnotation] == fakeAttachVolumeKey {
+		delete(volumeCR.Annotations, fakeAttachVolumeAnnotation)
+		ll.Warningf("Removing fake-attach annotation for volume %s", volumeID)
+		s.VolumeManager.recorder.Eventf(volumeCR, eventing.NormalType, eventing.FakeAttachCleared,
+			"Fake-attach cleared for volume with ID %s", volumeID)
 	}
 
 	if currStatus != apiV1.VolumeReady {
@@ -293,12 +303,11 @@ func (s *CSINodeService) NodeUnstageVolume(ctx context.Context, req *csi.NodeUns
 		errToReturn error
 	)
 
-	if volumeCR.Annotations[fakeAttachVolumeAnnotation] == fakeAttachVolumeKey {
-		ll.Warningf("Removing fake-attach annotation for volume %s", volumeID)
-		delete(volumeCR.Annotations, fakeAttachVolumeAnnotation)
-	} else if errToReturn = s.fsOps.UnmountWithCheck(getStagingPath(ll, req.GetStagingTargetPath())); errToReturn != nil {
-		volumeCR.Spec.CSIStatus = apiV1.Failed
-		resp = nil
+	if volumeCR.Annotations[fakeAttachVolumeAnnotation] != fakeAttachVolumeKey {
+		if errToReturn = s.fsOps.UnmountWithCheck(getStagingPath(ll, req.GetStagingTargetPath())); errToReturn != nil {
+			volumeCR.Spec.CSIStatus = apiV1.Failed
+			resp = nil
+		}
 	}
 
 	ctxWithID := context.WithValue(context.Background(), base.RequestUUID, req.GetVolumeId())
diff --git a/pkg/node/volumemgr.go b/pkg/node/volumemgr.go
index d9c5104..ae86c79 100644
--- a/pkg/node/volumemgr.go
+++ b/pkg/node/volumemgr.go
@@ -67,6 +67,10 @@ const (
 
 	fakeAttachAnnotation = "pv.attach.kubernetes.io/ignore-if-inaccessible"
 	fakeAttachAllowKey   = "yes"
+
+	// Annotation key for health overriding
+	// Discover function replaces drive health with passed value if the annotation is set
+	driveHealthOverrideAnnotation = "health"
 )
 
 // eventRecorder interface for sending events
@@ -602,6 +606,9 @@ func (m *VolumeManager) updateDrivesCRs(ctx context.Context, drivesFromMgr []*ap
 				if searchSystemDrives && driveCR.Spec.IsSystem {
 					m.systemDrivesUUIDs = append(m.systemDrivesUUIDs, driveCR.Spec.UUID)
 				}
+				if value, ok := driveCR.GetAnnotations()[driveHealthOverrideAnnotation]; ok {
+					m.overrideDriveHealth(drivePtr, value, driveCR.Name)
+				}
 				if driveCR.Equals(drivePtr) {
 					updates.AddNotChanged(&driveCR)
 				} else {
@@ -979,14 +986,20 @@ func (m *VolumeManager) createEventsForDriveUpdates(updates *driveUpdates) {
 			createdDrive, apiV1.HealthUnknown, createdDrive.Spec.Health)
 	}
 	for _, updDrive := range updates.Updated {
-		if updDrive.CurrentState.Spec.Health != updDrive.PreviousState.Spec.Health {
-			m.createEventForDriveHealthChange(
-				updDrive.CurrentState, updDrive.PreviousState.Spec.Health, updDrive.CurrentState.Spec.Health)
-		}
 		if updDrive.CurrentState.Spec.Status != updDrive.PreviousState.Spec.Status {
 			m.createEventForDriveStatusChange(
 				updDrive.CurrentState, updDrive.PreviousState.Spec.Status, updDrive.CurrentState.Spec.Status)
 		}
+		// skip HealthChange event if drive health was overridden
+		if _, ok := updDrive.CurrentState.Annotations[driveHealthOverrideAnnotation]; ok {
+			m.createEventForDriveHealthOverridden(
+				updDrive.CurrentState, updDrive.PreviousState.Spec.Health, updDrive.CurrentState.Spec.Health)
+			continue
+		}
+		if updDrive.CurrentState.Spec.Health != updDrive.PreviousState.Spec.Health {
+			m.createEventForDriveHealthChange(
+				updDrive.CurrentState, updDrive.PreviousState.Spec.Health, updDrive.CurrentState.Spec.Health)
+		}
 	}
 }
 
@@ -1031,6 +1044,16 @@ func (m *VolumeManager) createEventForDriveStatusChange(
 		statusMsgTemplate, currentStatus, prevStatus)
 }
 
+// createEventForDriveHealthOverridden creates DriveHealthOverridden with Warning type
+func (m *VolumeManager) createEventForDriveHealthOverridden(
+	drive *drivecrd.Drive, realHealth, overriddenHealth string) {
+	msgTemplate := "Drive health is overridden with: %s, real state: %s."
+	eventType := eventing.WarningType
+	reason := eventing.DriveHealthOverridden
+	m.sendEventForDrive(drive, eventType, reason,
+		msgTemplate, overriddenHealth, realHealth)
+}
+
 func (m *VolumeManager) sendEventForDrive(drive *drivecrd.Drive, eventtype, reason, messageFmt string,
 	args ...interface{}) {
 	messageFmt += drive.GetDriveDescription()
@@ -1136,3 +1159,19 @@ func (m *VolumeManager) isPVCNeedFakeAttach(volumeID string) bool {
 
 	return false
 }
+
+// overrideDriveHealth replaces drive health with passed value,
+// generates error message if value is not valid
+func (m *VolumeManager) overrideDriveHealth(drive *api.Drive, overriddenHealth, driveCRName string) {
+	if (overriddenHealth == apiV1.HealthGood) ||
+		(overriddenHealth == apiV1.HealthSuspect) ||
+		(overriddenHealth == apiV1.HealthBad) ||
+		(overriddenHealth == apiV1.HealthUnknown) {
+		m.log.Warnf("Drive %s has health annotation. Health %s has been overridden with %s.",
+			driveCRName, drive.Health, overriddenHealth)
+		drive.Health = overriddenHealth
+	} else {
+		m.log.Errorf("Drive %s has health annotation, but value %s is not %s/%s/%s/%s. Health is not overridden.",
+			driveCRName, overriddenHealth, apiV1.HealthGood, apiV1.HealthSuspect, apiV1.HealthBad, apiV1.HealthUnknown)
+	}
+}
diff --git a/pkg/scheduler/extender/extender.go b/pkg/scheduler/extender/extender.go
index f38a4f4..c9d667e 100644
--- a/pkg/scheduler/extender/extender.go
+++ b/pkg/scheduler/extender/extender.go
@@ -226,6 +226,7 @@ func (e *Extender) gatherCapacityRequestsByProvisioner(ctx context.Context, pod
 
 	requests := make([]*genV1.CapacityRequest, 0)
 	for _, v := range pod.Spec.Volumes {
+		ll.Debugf("Volume details: %s", v.String())
 		// check whether volume Ephemeral or not
 		if v.CSI != nil {
 			if v.CSI.Driver == e.provisioner {
diff --git a/variables.mk b/variables.mk
index 7d81331..99709fb 100644
--- a/variables.mk
+++ b/variables.mk
@@ -71,5 +71,10 @@ GOPROXY_PART    := GOPROXY=https://proxy.golang.org,direct
 METRICS_PACKAGE := github.com/dell/csi-baremetal/pkg/metrics
 LDFLAGS := -ldflags "-X ${METRICS_PACKAGE}.Revision=${RELEASE_STR} -X ${METRICS_PACKAGE}.Branch=${BRANCH}"
 
+### Kind
+KIND_DIR := test/kind
+KIND     := ${KIND_DIR}/kind
+KIND_VER := 0.8.1
+
 # override some of variables, optional file
 -include variables.override.mk


Test patch:
diff --git a/pkg/node/node_test.go b/pkg/node/node_test.go
index 373fb12..2e22039 100644
--- a/pkg/node/node_test.go
+++ b/pkg/node/node_test.go
@@ -683,7 +683,7 @@ var _ = Describe("CSINodeService Fake-Attach", func() {
 		setVariables()
 	})
 
-	It("Should stage volume with fake-attach annotation", func() {
+	It("Should stage unhealthy volume with fake-attach annotation", func() {
 		req := getNodeStageRequest(testVolume1.Id, *testVolumeCap)
 		vol1 := testVolumeCR1
 		vol1.Spec.CSIStatus = apiV1.Created
@@ -723,7 +723,7 @@ var _ = Describe("CSINodeService Fake-Attach", func() {
 		Expect(vol1.Spec.CSIStatus).To(Equal(apiV1.VolumeReady))
 		Expect(vol1.Annotations[fakeAttachVolumeAnnotation]).To(Equal(fakeAttachVolumeKey))
 	})
-	It("Should publish volume with fake-attach annotation", func() {
+	It("Should publish unhealthy volume with fake-attach annotation", func() {
 		req := getNodePublishRequest(testV1ID, targetPath, *testVolumeCap)
 		req.VolumeContext[util.PodNameKey] = testPodName
 
@@ -746,7 +746,48 @@ var _ = Describe("CSINodeService Fake-Attach", func() {
 		Expect(err).To(BeNil())
 		Expect(volumeCR.Spec.Owners[0]).To(Equal(testPodName))
 	})
-	It("Should unstage volume with fake-attach", func() {
+	It("Should stage healthy volume with fake-attach annotation", func() {
+		req := getNodeStageRequest(testVolume1.Id, *testVolumeCap)
+		vol1 := testVolumeCR1
+		vol1.Spec.CSIStatus = apiV1.Created
+		err := node.k8sClient.UpdateCR(testCtx, &vol1)
+		Expect(err).To(BeNil())
+
+		pvcName := "pvcName"
+		pvcNamespace := "pvcNamespace"
+
+		pv := &corev1.PersistentVolume{}
+		pv.Name = vol1.Name
+		pv.Spec.ClaimRef = &corev1.ObjectReference{}
+		pv.Spec.ClaimRef.Name = pvcName
+		pv.Spec.ClaimRef.Namespace = pvcNamespace
+		err = node.k8sClient.Create(testCtx, pv)
+		Expect(err).To(BeNil())
+
+		pvc := &corev1.PersistentVolumeClaim{}
+		pvc.Name = pvcName
+		pvc.Namespace = pvcNamespace
+		pvc.Annotations = map[string]string{fakeAttachAnnotation: fakeAttachAllowKey}
+		err = node.k8sClient.Create(testCtx, pvc)
+		Expect(err).To(BeNil())
+
+		partitionPath := "/partition/path/for/volume1"
+		prov.On("GetVolumePath", vol1.Spec).Return(partitionPath, nil)
+		fsOps.On("PrepareAndPerformMount",
+			partitionPath, path.Join(req.GetStagingTargetPath(), stagingFileName), true, false).
+			Return(nil)
+
+		resp, err := node.NodeStageVolume(testCtx, req)
+		Expect(resp).NotTo(BeNil())
+		Expect(err).To(BeNil())
+
+		err = node.k8sClient.ReadCR(testCtx, testV1ID, "", &vol1)
+		Expect(err).To(BeNil())
+		Expect(vol1.Spec.CSIStatus).To(Equal(apiV1.VolumeReady))
+		_, ok := vol1.Annotations[fakeAttachVolumeAnnotation]
+		Expect(ok).To(Equal(false))
+	})
+	It("Should unstage volume with fake-attach annotation", func() {
 		req := getNodeUnstageRequest(testV1ID, stagePath)
 
 		vol1 := testVolumeCR1
@@ -763,7 +804,7 @@ var _ = Describe("CSINodeService Fake-Attach", func() {
 		Expect(err).To(BeNil())
 		Expect(volumeCR.Spec.CSIStatus).To(Equal(apiV1.Created))
 		_, ok := volumeCR.Annotations[fakeAttachVolumeAnnotation]
-		Expect(ok).To(Equal(false))
+		Expect(ok).To(Equal(true))
 	})
 })
 
diff --git a/pkg/node/volumemgr_test.go b/pkg/node/volumemgr_test.go
index db63883..09a9742 100644
--- a/pkg/node/volumemgr_test.go
+++ b/pkg/node/volumemgr_test.go
@@ -778,6 +778,26 @@ func TestVolumeManager_updatesDrivesCRs_Fail(t *testing.T) {
 	assert.Equal(t, 1, len(res.Created))
 }
 
+func TestVolumeManager_updatesDrivesCRs_Override(t *testing.T) {
+	client, err := k8s.GetFakeKubeClient(testNs, testLogger)
+	assert.Nil(t, err)
+	vm := NewVolumeManager(nil, nil, testLogger, client, client, new(mocks.NoOpRecorder), nodeID)
+
+	testDriveCR1 := testDriveCR.DeepCopy()
+	testDriveCR1.Annotations = map[string]string{driveHealthOverrideAnnotation: apiV1.HealthSuspect}
+	err = client.CreateCR(testCtx, testDriveCR1.Name, testDriveCR1)
+	assert.Nil(t, err)
+
+	drive := drive1
+	drives := []*api.Drive{&drive}
+
+	updates, err := vm.updateDrivesCRs(testCtx, drives)
+	assert.Nil(t, err)
+	assert.Equal(t, len(updates.Updated), 1)
+	assert.Equal(t, updates.Updated[0].PreviousState.Spec.Health, drive1.Health)
+	assert.Equal(t, updates.Updated[0].CurrentState.Spec.Health, apiV1.HealthSuspect)
+}
+
 func TestVolumeManager_handleDriveStatusChange(t *testing.T) {
 	vm := prepareSuccessVolumeManagerWithDrives(nil, t)
 
diff --git a/test/e2e/common/deploy.go b/test/e2e/common/deploy.go
index c650cf5..30e4798 100644
--- a/test/e2e/common/deploy.go
+++ b/test/e2e/common/deploy.go
@@ -103,7 +103,7 @@ func DeployOperator(f *framework.Framework) (func(), error) {
 	return cleanup, nil
 }
 
-// DeployCSIWithArgs deploys csi-baremetal-deployment with CmdHelmExecutor
+// DeployCSI deploys csi-baremetal-deployment with CmdHelmExecutor
 // After install - waiting all pods ready, checking kubernetes-scheduler restart
 // Cleanup - deleting csi-chart, cleaning all csi custom resources
 // Helm command - "helm install csi-baremetal <CHARTS_DIR>/csi-baremetal-deployment
diff --git a/test/kind/kind-0.11.1.patch b/test/kind/kind-0.11.1.patch
new file mode 100644
index 0000000..32d7653
--- /dev/null
+++ b/test/kind/kind-0.11.1.patch
@@ -0,0 +1,22 @@
+--- kind-0.11.1/pkg/cluster/internal/providers/docker/provision.go	2021-07-02 15:32:23.872508147 +0300
++++ kind-0.11.1-patched/pkg/cluster/internal/providers/docker/provision.go	2021-06-30 18:57:28.806000000 +0300
+@@ -243,6 +243,8 @@
+ 		"--volume", "/var",
+ 		// some k8s things want to read /lib/modules
+ 		"--volume", "/lib/modules:/lib/modules:ro",
++		// ipc host
++		"--ipc", "host",
+ 	},
+ 		args...,
+ 	)
+--- kind-0.11.1/pkg/cmd/kind/version/version.go	2021-05-28 02:18:13.000000000 +0300
++++ kind-0.11.1-patched/pkg/cmd/kind/version/version.go	2021-07-02 15:31:45.440367546 +0300
+@@ -50,7 +50,7 @@
+ }
+ 
+ // VersionCore is the core portion of the kind CLI version per Semantic Versioning 2.0.0
+-const VersionCore = "0.11.1"
++const VersionCore = "0.11.1-patched"
+ 
+ // VersionPreRelease is the pre-release portion of the kind CLI version per
+ // Semantic Versioning 2.0.0
diff --git a/test/kind/kind-0.8.1.patch b/test/kind/kind-0.8.1.patch
new file mode 100644
index 0000000..e513303
--- /dev/null
+++ b/test/kind/kind-0.8.1.patch
@@ -0,0 +1,22 @@
+--- kind-0.8.1/pkg/cluster/internal/providers/docker/provision.go	2020-05-02 02:54:15.000000000 +0300
++++ kind-0.8.1/pkg/cluster/internal/providers/docker/provision-patched.go	2021-07-06 12:36:23.376743088 +0300
+@@ -227,6 +227,8 @@
+ 		"--volume", "/var",
+ 		// some k8s things want to read /lib/modules
+ 		"--volume", "/lib/modules:/lib/modules:ro",
++		// ipc host
++		"--ipc", "host",
+ 	},
+ 		args...,
+ 	)
+--- kind-0.8.1/pkg/cmd/kind/version/version.go	2020-05-02 02:54:15.000000000 +0300
++++ kind-0.8.1/pkg/cmd/kind/version/version-patched.go	2021-07-06 12:38:29.521715917 +0300
+@@ -50,7 +50,7 @@
+ }
+ 
+ // VersionCore is the core portion of the kind CLI version per Semantic Versioning 2.0.0
+-const VersionCore = "0.8.1"
++const VersionCore = "0.8.1-patched"
+ 
+ // VersionPreRelease is the pre-release portion of the kind CLI version per
+ // Semantic Versioning 2.0.0