Repository: dell/csi-baremetal
Pull request #554 (2021-09-28 08:46:23)

[ISSUE-551] Support proactive removal for OFFLINE drive
## Purpose
### Issue #551 

- fix incorrect behavior on `health=bad` annotation for OFFLINE drive
- skip partition deletion if volume in MISSING
- locate Node instead of drive if it is OFFLINE

## PR checklist
- [x] Add link to the issue
- [x] Choose PR label
- [ ] New unit tests added
- [ ] Modified code has meaningful comments
- [ ] All TODOs are linked with the issues
- [ ] All comments are resolved

## Testing
```
# Pvc bound, volume created
[root@servicenode ~]# kubectl get volume
NAME                                       SIZE            STORAGE CLASS   HEALTH   CSI_STATUS   LOCATION                               NODE
pvc-25d3431e-03d5-48d1-9bf8-0cfa9a3c980d   8001000000000   HDD             GOOD     PUBLISHED    e1d1dfed-22bb-463e-95fa-7a32d49abd60   85cf358f-3b5b-4c76-961d-9fb6f0e2ebe8
[root@servicenode ~]# kubectl get pvc
NAME         STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS       AGE
www-webi-0   Bound    pvc-25d3431e-03d5-48d1-9bf8-0cfa9a3c980d   7452Gi     RWO            csi-baremetal-sc   87s
[root@servicenode ~]# kubectl get drive -o wide | grep e1d1dfed-22bb-463e-95fa-7a32d49abd60
e1d1dfed-22bb-463e-95fa-7a32d49abd60   8001000000000   HDD    GOOD     ONLINE    IN_USE            /dev/sdh   VDH147TD           17     85cf358f-3b5b-4c76-961d-9fb6f0e2ebe8

# Disk goes to OFFLINE
[root@servicenode ~]# kubectl get drive -o wide | grep e1d1dfed-22bb-463e-95fa-7a32d49abd60
e1d1dfed-22bb-463e-95fa-7a32d49abd60   8001000000000   HDD    UNKNOWN   OFFLINE   IN_USE            /dev/sdh   VDH147TD           17     85cf358f-3b5b-4c76-961d-9fb6f0e2ebe8
[root@servicenode ~]# kubectl get volume
NAME                                       SIZE            STORAGE CLASS   HEALTH    CSI_STATUS   LOCATION                               NODE
pvc-25d3431e-03d5-48d1-9bf8-0cfa9a3c980d   8001000000000   HDD             UNKNOWN   PUBLISHED    e1d1dfed-22bb-463e-95fa-7a32d49abd60   85cf358f-3b5b-4c76-961d-9fb6f0e2ebe8

# Annotate drive, wait DR start
[root@servicenode ~]# kubectl annotate drive e1d1dfed-22bb-463e-95fa-7a32d49abd60 health=bad
drive.csi-baremetal.dell.com/e1d1dfed-22bb-463e-95fa-7a32d49abd60 annotated
[root@servicenode ~]# kubectl get drive -o wide | grep e1d1dfed-22bb-463e-95fa-7a32d49abd60
e1d1dfed-22bb-463e-95fa-7a32d49abd60   8001000000000   HDD    BAD      OFFLINE   RELEASING            /dev/sdh   VDH147TD           17     85cf358f-3b5b-4c76-961d-9fb6f0e2ebe8

# Perform disk removal
[root@servicenode ~]# kubectl annotate volume pvc-25d3431e-03d5-48d1-9bf8-0cfa9a3c980d release=done
volume.csi-baremetal.dell.com/pvc-25d3431e-03d5-48d1-9bf8-0cfa9a3c980d annotated
[root@servicenode ~]# kubectl get volume -o wide
NAME                                       SIZE            STORAGE CLASS   HEALTH   CSI_STATUS   OP_STATUS   USAGE      TYPE   LOCATION                               NODE
pvc-25d3431e-03d5-48d1-9bf8-0cfa9a3c980d   8001000000000   HDD             BAD      PUBLISHED    MISSING     RELEASED   xfs    e1d1dfed-22bb-463e-95fa-7a32d49abd60   85cf358f-3b5b-4c76-961d-9fb6f0e2ebe8
[root@servicenode ~]# kubectl get drive -o wide | grep e1d1dfed-22bb-463e-95fa-7a32d49abd60
e1d1dfed-22bb-463e-95fa-7a32d49abd60   8001000000000   HDD    BAD      OFFLINE   RELEASED            /dev/sdh   VDH147TD           17     85cf358f-3b5b-4c76-961d-9fb6f0e2ebe8
[root@servicenode ~]# kubectl annotate drive e1d1dfed-22bb-463e-95fa-7a32d49abd60 removal=ready
drive.csi-baremetal.dell.com/e1d1dfed-22bb-463e-95fa-7a32d49abd60 annotated
[root@servicenode ~]# kubectl get drive -o wide | grep e1d1dfed-22bb-463e-95fa-7a32d49abd60
e1d1dfed-22bb-463e-95fa-7a32d49abd60   8001000000000   HDD    BAD      OFFLINE   REMOVING            /dev/sdh   VDH147TD           17     85cf358f-3b5b-4c76-961d-9fb6f0e2ebe8
[root@servicenode ~]# kubectl delete pvc www-webi-0 --wait=false
persistentvolumeclaim "www-webi-0" deleted
[root@servicenode ~]# kubectl delete po webi-0
pod "webi-0" deleted

# Disk removed
[root@servicenode ~]# kubectl get drive -o wide e1d1dfed-22bb-463e-95fa-7a32d49abd60
Error from server (NotFound): drives.csi-baremetal.dell.com "e1d1dfed-22bb-463e-95fa-7a32d49abd60" not found

# Pod running, pvc bound
[root@servicenode ~]# kubectl get pvc
NAME         STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS       AGE
www-webi-0   Bound    pvc-fd4b9b9f-b1d5-46c3-a1e0-4267f634eb0e   7452Gi     RWO            csi-baremetal-sc   12m
[root@servicenode ~]# kubectl get po webi-0
NAME     READY   STATUS    RESTARTS   AGE
webi-0   1/1     Running   0          13m
```


codecov[bot]: # [Codecov](https://codecov.io/gh/dell/csi-baremetal/pull/554?src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=dell) Report
> Merging [#554](https://codecov.io/gh/dell/csi-baremetal/pull/554?src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=dell) (0e6ce43) into [master](https://codecov.io/gh/dell/csi-baremetal/commit/f1e770731408c36a0fad63dcc211a68d350ebb4d?el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=dell) (f1e7707) will **decrease** coverage by `0.03%`.
> The diff coverage is `67.74%`.

> :exclamation: Current head 0e6ce43 differs from pull request most recent head bea8e97. Consider uploading reports for the commit bea8e97 to get more accurate results
[![Impacted file tree graph](https://codecov.io/gh/dell/csi-baremetal/pull/554/graphs/tree.svg?width=650&height=150&src=pr&token=4D8WVNWX5O&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=dell)](https://codecov.io/gh/dell/csi-baremetal/pull/554?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=dell)

```diff
@@            Coverage Diff             @@
##           master     #554      +/-   ##
==========================================
- Coverage   60.32%   60.29%   -0.04%     
==========================================
  Files          60       60              
  Lines        6017     6027      +10     
==========================================
+ Hits         3630     3634       +4     
- Misses       2174     2176       +2     
- Partials      213      217       +4     
```

| Flag | Coverage Δ | |
|---|---|---|
| unittests | `60.29% <67.74%> (-0.04%)` | :arrow_down: |

Flags with carried forward coverage won't be shown. [Click here](https://docs.codecov.io/docs/carryforward-flags?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=dell#carryforward-flags-in-the-pull-request-comment) to find out more.

| [Impacted Files](https://codecov.io/gh/dell/csi-baremetal/pull/554?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=dell) | Coverage Δ | |
|---|---|---|
| [pkg/node/volumemgr.go](https://codecov.io/gh/dell/csi-baremetal/pull/554/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=dell#diff-cGtnL25vZGUvdm9sdW1lbWdyLmdv) | `67.87% <67.74%> (-0.42%)` | :arrow_down: |

------

[Continue to review full report at Codecov](https://codecov.io/gh/dell/csi-baremetal/pull/554?src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=dell).
> **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=dell)
> `Δ = absolute <relative> (impact)`, `ø = not affected`, `? = missing data`
> Powered by [Codecov](https://codecov.io/gh/dell/csi-baremetal/pull/554?src=pr&el=footer&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=dell). Last update [f1e7707...bea8e97](https://codecov.io/gh/dell/csi-baremetal/pull/554?src=pr&el=lastupdated&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=dell). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=dell).


Patch:
diff --git a/pkg/crcontrollers/drive/drivecontroller.go b/pkg/crcontrollers/drive/drivecontroller.go
index 76e1cc1..afcab48 100644
--- a/pkg/crcontrollers/drive/drivecontroller.go
+++ b/pkg/crcontrollers/drive/drivecontroller.go
@@ -207,20 +207,18 @@ func (c *Controller) handleDriveUpdate(ctx context.Context, log *logrus.Entry, d
 		}
 		fallthrough
 	case apiV1.DriveUsageRemoving:
-		volumes, err := c.crHelper.GetVolumesByLocation(ctx, id)
+		status, err := c.handleDriveUsageRemoving(ctx, log, drive)
 		if err != nil {
-			return ignore, err
+			return status, err
 		}
-		if c.checkAllVolsRemoved(volumes) {
-			drive.Spec.Usage = apiV1.DriveUsageRemoved
-			c.locateDriveLED(ctx, log, drive)
+		if status == update {
 			toUpdate = true
 		}
 	case apiV1.DriveUsageRemoved:
 		if drive.Spec.Status == apiV1.DriveStatusOffline {
 			// drive was removed from the system. need to clean corresponding custom resource
 			// try to stop node LED
-			if err := c.locateNodeLED(ctx, log, drive); err != nil {
+			if err := c.stopLocateNodeLED(ctx, log, drive); err != nil {
 				return ignore, err
 			}
 			return remove, nil
@@ -321,11 +319,33 @@ func (c *Controller) checkAndPlaceStatusRemoved(drive *drivecrd.Drive) bool {
 	return false
 }
 
+func (c *Controller) handleDriveUsageRemoving(ctx context.Context, log *logrus.Entry, drive *drivecrd.Drive) (uint8, error) {
+	volumes, err := c.crHelper.GetVolumesByLocation(ctx, drive.Spec.UUID)
+	if err != nil {
+		return ignore, err
+	}
+	if !c.checkAllVolsRemoved(volumes) {
+		return ignore, nil
+	}
+	drive.Spec.Usage = apiV1.DriveUsageRemoved
+	if drive.Spec.Status == apiV1.DriveStatusOnline {
+		c.locateDriveLED(ctx, log, drive)
+	} else {
+		// We can not set locate for missing disks, try to locate Node instead
+		log.Infof("Try to locate node LED %s", drive.Spec.NodeId)
+		if _, locateErr := c.driveMgrClient.LocateNode(ctx, &api.NodeLocateRequest{Action: apiV1.LocateStart}); locateErr != nil {
+			log.Errorf("Failed to start node locate: %s", err.Error())
+			return ignore, err
+		}
+	}
+	return update, nil
+}
+
 func (c *Controller) locateDriveLED(ctx context.Context, log *logrus.Entry, drive *drivecrd.Drive) {
 	// try to enable LED
 	status, err := c.driveMgrClient.Locate(ctx, &api.DriveLocateRequest{Action: apiV1.LocateStart, DriveSerialNumber: drive.Spec.SerialNumber})
 	if err != nil || status.Status != apiV1.LocateStatusOn {
-		log.Errorf("Failed to locate LED of drive %s, err %v", drive.Spec.SerialNumber, err)
+		log.Errorf("Failed to locate LED of drive %s, LED status - %d, err %v", drive.Spec.SerialNumber, status.Status, err)
 		drive.Spec.Usage = apiV1.DriveUsageFailed
 		// send error level alert
 		eventMsg := fmt.Sprintf("Failed to locale LED, %s", drive.GetDriveDescription())
@@ -337,7 +357,7 @@ func (c *Controller) locateDriveLED(ctx context.Context, log *logrus.Entry, driv
 	}
 }
 
-func (c *Controller) locateNodeLED(ctx context.Context, log *logrus.Entry, curDrive *drivecrd.Drive) error {
+func (c *Controller) stopLocateNodeLED(ctx context.Context, log *logrus.Entry, curDrive *drivecrd.Drive) error {
 	driveList := &drivecrd.DriveList{}
 	if err := c.client.ReadList(ctx, driveList); err != nil {
 		log.Errorf("Unable to read Drive List")
diff --git a/pkg/node/volumemgr.go b/pkg/node/volumemgr.go
index 1c18742..8961336 100644
--- a/pkg/node/volumemgr.go
+++ b/pkg/node/volumemgr.go
@@ -474,33 +474,47 @@ func (m *VolumeManager) handleRemovingStatus(ctx context.Context, volume *volume
 		"volumeID": volume.Name,
 	})
 
-	var (
-		err       error
-		newStatus string
-	)
-	if err = m.getProvisionerForVolume(&volume.Spec).ReleaseVolume(volume.Spec); err != nil {
+	newStatus, err := m.performVolumeRemoving(ctx, volume)
+	if err != nil && newStatus == "" {
+		return ctrl.Result{Requeue: true}, err
+	}
+
+	volume.Spec.CSIStatus = newStatus
+	if updateErr := m.k8sClient.UpdateCRWithAttempts(ctx, volume, 10); updateErr != nil {
+		ll.Error("Unable to set new status for volume")
+		return ctrl.Result{Requeue: true}, updateErr
+	}
+	return ctrl.Result{}, err
+}
+
+func (m *VolumeManager) performVolumeRemoving(ctx context.Context, volume *volumecrd.Volume) (string, error) {
+	ll := m.log.WithFields(logrus.Fields{
+		"method":   "performVolumeRemoving",
+		"volumeID": volume.Name,
+	})
+
+	if volume.Spec.GetOperationalStatus() == apiV1.OperationalStatusMissing {
+		ll.Warnf("Volume - %s is MISSING. Unable to perform deletion. Set status to Removed", volume.Spec.Id)
+		return apiV1.Removed, nil
+	}
+
+	if err := m.getProvisionerForVolume(&volume.Spec).ReleaseVolume(volume.Spec); err != nil {
 		ll.Errorf("Failed to remove volume - %s. Error: %v. Set status to Failed", volume.Spec.Id, err)
-		newStatus = apiV1.Failed
 		drive := m.crHelper.GetDriveCRByUUID(volume.Spec.Location)
 		if drive != nil {
 			drive.Spec.Usage = apiV1.DriveUsageFailed
 			if err := m.k8sClient.UpdateCRWithAttempts(ctx, drive, 5); err != nil {
 				ll.Errorf("Unable to change drive %s usage status to %s, error: %v.",
 					drive.Name, drive.Spec.Usage, err)
-				return ctrl.Result{Requeue: true}, err
+				return "", err
 			}
 			m.sendEventForDrive(drive, eventing.DriveRemovalFailed, deleteVolumeFailedMsg, volume.Name, err)
 		}
-	} else {
-		ll.Infof("Volume - %s was successfully removed. Set status to Removed", volume.Spec.Id)
-		newStatus = apiV1.Removed
+		return apiV1.Failed, err
 	}
-	volume.Spec.CSIStatus = newStatus
-	if updateErr := m.k8sClient.UpdateCRWithAttempts(ctx, volume, 10); updateErr != nil {
-		ll.Error("Unable to set new status for volume")
-		return ctrl.Result{Requeue: true}, updateErr
-	}
-	return ctrl.Result{}, err
+
+	ll.Infof("Volume - %s was successfully removed. Set status to Removed", volume.Spec.Id)
+	return apiV1.Removed, nil
 }
 
 // SetupWithManager registers VolumeManager to ControllerManager
@@ -688,7 +702,11 @@ func (m *VolumeManager) updateDrivesCRs(ctx context.Context, drivesFromMgr []*ap
 			toUpdate := d
 			// TODO: which operational status should be in case when there is drive CR that doesn't have corresponding drive from drivemgr response
 			toUpdate.Spec.Status = apiV1.DriveStatusOffline
-			toUpdate.Spec.Health = apiV1.HealthUnknown
+			if value, ok := d.GetAnnotations()[driveHealthOverrideAnnotation]; ok {
+				m.overrideDriveHealth(&toUpdate.Spec, value, d.Name)
+			} else {
+				toUpdate.Spec.Health = apiV1.HealthUnknown
+			}
 			if err := m.k8sClient.UpdateCR(ctx, &toUpdate); err != nil {
 				ll.Errorf("Failed to update drive CR %v, error %v", toUpdate, err)
 				updates.AddNotChanged(previousState)
@@ -1020,11 +1038,11 @@ func (m *VolumeManager) createEventsForDriveUpdates(updates *driveUpdates) {
 			m.createEventForDriveStatusChange(
 				updDrive.CurrentState, updDrive.PreviousState.Spec.Status, updDrive.CurrentState.Spec.Status)
 		}
-		if _, ok := updDrive.CurrentState.Annotations[driveHealthOverrideAnnotation]; ok {
-			m.createEventForDriveHealthOverridden(
-				updDrive.CurrentState, updDrive.PreviousState.Spec.Health, updDrive.CurrentState.Spec.Health)
-		}
 		if updDrive.CurrentState.Spec.Health != updDrive.PreviousState.Spec.Health {
+			if _, ok := updDrive.CurrentState.Annotations[driveHealthOverrideAnnotation]; ok {
+				m.createEventForDriveHealthOverridden(
+					updDrive.CurrentState, updDrive.PreviousState.Spec.Health, updDrive.CurrentState.Spec.Health)
+			}
 			m.createEventForDriveHealthChange(
 				updDrive.CurrentState, updDrive.PreviousState.Spec.Health, updDrive.CurrentState.Spec.Health)
 		}


Test patch:
diff --git a/pkg/node/volumemgr_test.go b/pkg/node/volumemgr_test.go
index f651633..1a8048e 100644
--- a/pkg/node/volumemgr_test.go
+++ b/pkg/node/volumemgr_test.go
@@ -143,7 +143,7 @@ var (
 			APIVersion: apiV1.APIV1Version,
 		},
 		ObjectMeta: v1.ObjectMeta{
-			Name:      testLVGName,
+			Name: testLVGName,
 		},
 		Spec: api.LogicalVolumeGroup{
 			Name:       testLVGName,
@@ -315,44 +315,65 @@ func TestVolumeManager_handleRemovingStatus(t *testing.T) {
 		err    error
 	)
 
-	// happy path
-	vm = prepareSuccessVolumeManager(t)
-	testVol := volCR.DeepCopy()
-	testVol.Spec.CSIStatus = apiV1.Removing
-	assert.Nil(t, vm.k8sClient.CreateCR(testCtx, testVol.Name, testVol))
-	pMock := mockProv.GetMockProvisionerSuccess("/some/path")
-	vm.SetProvisioners(map[p.VolumeType]p.Provisioner{p.DriveBasedVolumeType: pMock})
+	t.Run("happy path", func(t *testing.T) {
+		vm = prepareSuccessVolumeManager(t)
+		testVol := volCR.DeepCopy()
+		testVol.Spec.CSIStatus = apiV1.Removing
+		assert.Nil(t, vm.k8sClient.CreateCR(testCtx, testVol.Name, testVol))
+		pMock := mockProv.GetMockProvisionerSuccess("/some/path")
+		vm.SetProvisioners(map[p.VolumeType]p.Provisioner{p.DriveBasedVolumeType: pMock})
 
-	res, err = vm.handleRemovingStatus(testCtx, testVol)
-	assert.Nil(t, err)
-	assert.Equal(t, res, ctrl.Result{})
-	err = vm.k8sClient.ReadCR(testCtx, req.Name, testNs, volume)
-	assert.Nil(t, err)
-	assert.Equal(t, apiV1.Removed, volume.Spec.CSIStatus)
+		res, err = vm.handleRemovingStatus(testCtx, testVol)
+		assert.Nil(t, err)
+		assert.Equal(t, res, ctrl.Result{})
+		err = vm.k8sClient.ReadCR(testCtx, req.Name, testNs, volume)
+		assert.Nil(t, err)
+		assert.Equal(t, apiV1.Removed, volume.Spec.CSIStatus)
+	})
 
-	// failed to update
-	vm = prepareSuccessVolumeManager(t)
-	vm.SetProvisioners(map[p.VolumeType]p.Provisioner{p.DriveBasedVolumeType: pMock})
+	t.Run("failed to update", func(t *testing.T) {
+		testVol := volCR.DeepCopy()
+		vm = prepareSuccessVolumeManager(t)
+		pMock := mockProv.GetMockProvisionerSuccess("/some/path")
+		vm.SetProvisioners(map[p.VolumeType]p.Provisioner{p.DriveBasedVolumeType: pMock})
 
-	res, err = vm.handleRemovingStatus(testCtx, testVol)
-	assert.NotNil(t, err)
-	assert.True(t, res.Requeue)
+		res, err = vm.handleRemovingStatus(testCtx, testVol)
+		assert.NotNil(t, err)
+		assert.True(t, res.Requeue)
+	})
 
-	// ReleaseVolume failed
-	testVol = volCR.DeepCopy()
-	testVol.Spec.CSIStatus = apiV1.Removing
-	assert.Nil(t, vm.k8sClient.CreateCR(testCtx, testVol.Name, testVol))
-	pMock = &mockProv.MockProvisioner{}
-	pMock.On("ReleaseVolume", testVol.Spec).Return(testErr)
-	vm.SetProvisioners(map[p.VolumeType]p.Provisioner{p.DriveBasedVolumeType: pMock})
+	t.Run("ReleaseVolume failed", func(t *testing.T) {
+		testVol := volCR.DeepCopy()
+		testVol.Spec.CSIStatus = apiV1.Removing
+		assert.Nil(t, vm.k8sClient.CreateCR(testCtx, testVol.Name, testVol))
+		pMock := &mockProv.MockProvisioner{}
+		pMock.On("ReleaseVolume", testVol.Spec).Return(testErr)
+		vm.SetProvisioners(map[p.VolumeType]p.Provisioner{p.DriveBasedVolumeType: pMock})
 
-	res, err = vm.handleRemovingStatus(testCtx, testVol)
-	assert.NotNil(t, err)
-	assert.Equal(t, res, ctrl.Result{})
-	err = vm.k8sClient.ReadCR(testCtx, req.Name, testNs, volume)
-	assert.Nil(t, err)
-	assert.Equal(t, volume.Spec.CSIStatus, apiV1.Failed)
+		res, err = vm.handleRemovingStatus(testCtx, testVol)
+		assert.NotNil(t, err)
+		assert.Equal(t, res, ctrl.Result{})
+		err = vm.k8sClient.ReadCR(testCtx, req.Name, testNs, volume)
+		assert.Nil(t, err)
+		assert.Equal(t, volume.Spec.CSIStatus, apiV1.Failed)
+	})
 
+	t.Run("Volume missing", func(t *testing.T) {
+		vm = prepareSuccessVolumeManager(t)
+		testVol := volCR.DeepCopy()
+		testVol.Spec.CSIStatus = apiV1.Removing
+		testVol.Spec.OperationalStatus = apiV1.OperationalStatusMissing
+		assert.Nil(t, vm.k8sClient.CreateCR(testCtx, testVol.Name, testVol))
+		pMock := &mockProv.MockProvisioner{}
+		vm.SetProvisioners(map[p.VolumeType]p.Provisioner{p.DriveBasedVolumeType: pMock})
+
+		res, err = vm.handleRemovingStatus(testCtx, testVol)
+		assert.Nil(t, err)
+		assert.Equal(t, res, ctrl.Result{})
+		err = vm.k8sClient.ReadCR(testCtx, req.Name, testNs, volume)
+		assert.Nil(t, err)
+		assert.Equal(t, volume.Spec.CSIStatus, apiV1.Removed)
+	})
 }
 
 func TestVolumeManager_handleRemovingStatus_DeleteVolume(t *testing.T) {
@@ -711,6 +732,17 @@ func TestVolumeManager_updatesDrivesCRs_Success(t *testing.T) {
 	assert.Len(t, updates.Updated, 1)
 	assert.Len(t, updates.NotChanged, 1)
 
+	drives = driveMgrRespDrives[1:]
+	driveCRs, err = vm.crHelper.GetDriveCRs(vm.nodeID)
+	driveCRs[0].Annotations = map[string]string{"health": "bad"}
+	vm.k8sClient.UpdateCR(testCtx, &driveCRs[0])
+	updates, err = vm.updateDrivesCRs(testCtx, drives)
+	assert.Nil(t, err)
+	assert.Equal(t, vm.crHelper.GetDriveCRByUUID(driveMgrRespDrives[0].UUID).Spec.Health, apiV1.HealthBad)
+	assert.Equal(t, vm.crHelper.GetDriveCRByUUID(driveMgrRespDrives[0].UUID).Spec.Status, apiV1.DriveStatusOffline)
+	assert.Len(t, updates.Updated, 1)
+	assert.Len(t, updates.NotChanged, 1)
+
 	vm = prepareSuccessVolumeManager(t)
 	driveCRs, err = vm.crHelper.GetDriveCRs(vm.nodeID)
 	assert.Nil(t, err)
@@ -1021,6 +1053,22 @@ func TestVolumeManager_createEventsForDriveUpdates(t *testing.T) {
 		assert.True(t, expectEvent(drive1CR, eventing.DriveHealthUnknown))
 	})
 
+	t.Run("Drive health overriden", func(t *testing.T) {
+		init()
+		modifiedDrive := drive1CR.DeepCopy()
+		modifiedDrive.Annotations = map[string]string{"health": "bad"}
+		modifiedDrive.Spec.Health = apiV1.HealthBad
+
+		upd := &driveUpdates{
+			Updated: []updatedDrive{{
+				PreviousState: drive1CR,
+				CurrentState:  modifiedDrive}},
+		}
+		mgr.createEventsForDriveUpdates(upd)
+		assert.True(t, expectEvent(drive1CR, eventing.DriveHealthOverridden))
+		assert.True(t, expectEvent(drive1CR, eventing.DriveHealthFailure))
+	})
+
 	t.Run("Drive removed", func(t *testing.T) {
 		init()
 		modifiedDrive := drive1CR.DeepCopy()
@@ -1042,7 +1090,7 @@ func TestVolumeManager_createEventsForDriveUpdates(t *testing.T) {
 func TestVolumeManager_isShouldBeReconciled(t *testing.T) {
 	var (
 		vm  *VolumeManager = prepareSuccessVolumeManager(t)
-		vol *vcrd.Volume = testVolumeCR1.DeepCopy()
+		vol *vcrd.Volume   = testVolumeCR1.DeepCopy()
 	)
 
 	vol.Spec.NodeId = vm.nodeID