Repository: dell/csi-baremetal
Pull request #487 (2021-08-13 08:59:06)

[ISSUE-481] disk replacement events
## Purpose
### Issue #481

Move and rename DriveSuccessfullyReplacement event to DriveSuccessfullyRemoved.

## PR checklist
- [x] Add link to the issue
- [x] Choose PR label
- [x] New unit tests added
- [ ] Modified code has meaningful comments
- [ ] All 

## Testing
- Choose disk:
```
[root@servicenode ~]# kubectl get pvc data-web-3
NAME         STATUS   VOLUME                                     CAPACITY   ACCESS MODES   STORAGECLASS       AGE
data-web-3   Bound    pvc-f10e322e-2996-4bdf-81ce-10d212aefd9f   7452Gi     RWO            csi-baremetal-sc   98s
[root@servicenode ~]# kubectl get volume pvc-f10e322e-2996-4bdf-81ce-10d212aefd9f -o custom-columns-file=volume.txt
NAME                                       HEALTH   USAGE    STATUS      LOCATION
pvc-f10e322e-2996-4bdf-81ce-10d212aefd9f   GOOD     IN_USE   OPERATIVE   2e183ae5-2497-424e-bf0e-603d4e134757
[root@servicenode ~]# kubectl get drive  2e183ae5-2497-424e-bf0e-603d4e134757 -o custom-columns-file=drive.txt
NAME                                   HEALTH   USAGE    STATUS   TYPE
2e183ae5-2497-424e-bf0e-603d4e134757   GOOD     IN_USE   ONLINE   HDD
```
- Move disk to SUSPECT status:
```
[root@servicenode ~]# kubectl annotate drive 2e183ae5-2497-424e-bf0e-603d4e134757 health=suspect
drive.csi-baremetal.dell.com/2e183ae5-2497-424e-bf0e-603d4e134757 annotated

[root@servicenode ~]# kubectl get volume pvc-f10e322e-2996-4bdf-81ce-10d212aefd9f -o custom-columns-file=volume.txt
NAME                                       HEALTH    USAGE       STATUS      LOCATION
pvc-f10e322e-2996-4bdf-81ce-10d212aefd9f   SUSPECT   RELEASING   OPERATIVE   2e183ae5-2497-424e-bf0e-603d4e134757
[root@servicenode ~]# kubectl get drive  2e183ae5-2497-424e-bf0e-603d4e134757 -o custom-columns-file=drive.txt
NAME                                   HEALTH    USAGE       STATUS   TYPE
2e183ae5-2497-424e-bf0e-603d4e134757   SUSPECT   RELEASING   ONLINE   HDD
```
- Annotate volume release=done:
```
[root@servicenode ~]# kubectl annotate volume pvc-f10e322e-2996-4bdf-81ce-10d212aefd9f release=done
volume.csi-baremetal.dell.com/pvc-f10e322e-2996-4bdf-81ce-10d212aefd9f annotated

[root@servicenode ~]# kubectl get volume pvc-f10e322e-2996-4bdf-81ce-10d212aefd9f -o custom-columns-file=volume.txt
NAME                                       HEALTH    USAGE      STATUS      LOCATION
pvc-f10e322e-2996-4bdf-81ce-10d212aefd9f   SUSPECT   RELEASED   OPERATIVE   2e183ae5-2497-424e-bf0e-603d4e134757
[root@servicenode ~]# kubectl get drive  2e183ae5-2497-424e-bf0e-603d4e134757 -o custom-columns-file=drive.txt
NAME                                   HEALTH    USAGE      STATUS   TYPE
2e183ae5-2497-424e-bf0e-603d4e134757   SUSPECT   RELEASED   ONLINE   HDD
```
- Annotate drive replacement=ready:
```
[root@servicenode ~]# kubectl annotate drive 2e183ae5-2497-424e-bf0e-603d4e134757 replacement=ready
drive.csi-baremetal.dell.com/2e183ae5-2497-424e-bf0e-603d4e134757 annotated

[root@servicenode ~]# kubectl get volume pvc-f10e322e-2996-4bdf-81ce-10d212aefd9f -o custom-columns-file=volume.txt
NAME                                       HEALTH    USAGE      STATUS      LOCATION
pvc-f10e322e-2996-4bdf-81ce-10d212aefd9f   SUSPECT   RELEASED   OPERATIVE   2e183ae5-2497-424e-bf0e-603d4e134757
[root@servicenode ~]# kubectl get drive  2e183ae5-2497-424e-bf0e-603d4e134757 -o custom-columns-file=drive.txt
NAME                                   HEALTH    USAGE      STATUS   TYPE
2e183ae5-2497-424e-bf0e-603d4e134757   SUSPECT   REMOVING   ONLINE   HDD
```
- Remove pvc and pod:  
```
[root@servicenode ~]# kubectl delete pvc www-web-3 --wait=false
persistentvolumeclaim "www-web-3" deleted
[root@servicenode ~]# kubectl delete pod web-3
pod "web-3" deleted

[root@servicenode ~]# kubectl get volume pvc-f10e322e-2996-4bdf-81ce-10d212aefd9f -o custom-columns-file=volume.txt
Error from server (NotFound): volumes.csi-baremetal.dell.com "pvc-f10e322e-2996-4bdf-81ce-10d212aefd9f" not found
[root@servicenode ~]# kubectl get drive  2e183ae5-2497-424e-bf0e-603d4e134757 -o custom-columns-file=drive.txt
NAME                                   HEALTH    USAGE     STATUS   TYPE
2e183ae5-2497-424e-bf0e-603d4e134757   SUSPECT   REMOVED   ONLINE   HDD
```
- Remove drive:
```
[root@servicenode ~]# kubectl describe drive  2e183ae5-2497-424e-bf0e-603d4e134757 | grep "Node Id"
  Node Id:        6c0c897d-26e1-4b33-95ba-2af18970c072

[root@servicenode ~]# kubectl get csibmnode | grep 6c0c897d-26e1-4b33-95ba-2af18970c072
csibmnode-6c0c897d-26e1-4b33-95ba-2af18970c072   6c0c897d-26e1-4b33-95ba-2af18970c072   {"Hostname":"worker0.ocp4.atlantic.com","InternalIP":"10.245.137.190"}

[core@worker0 ~]$ echo 1 | sudo tee -a /sys/block/sdy/device/delete

[root@servicenode ~]# kubectl describe drive  2e183ae5-2497-424e-bf0e-603d4e134757
Error from server (NotFound): drives.csi-baremetal.dell.com "2e183ae5-2497-424e-bf0e-603d4e134757" not found
```

Generated events:
```
14m         Warning   DriveHealthOverridden          drive/2e183ae5-2497-424e-bf0e-603d4e134757                   Drive health is overridden with: SUSPECT, real state: GOOD. Drive Details: SN='VDGZPDGD', Node='6c0c897d-26e1-4b33-95ba-2af18970c072', Type='HDD', Model='ATA HGST HUS728T8TAL', Size='8001000000000', Firmware=''
14m         Warning   DriveHealthSuspect             drive/2e183ae5-2497-424e-bf0e-603d4e134757                   Drive health is: SUSPECT, previous state: GOOD. Drive Details: SN='VDGZPDGD', Node='6c0c897d-26e1-4b33-95ba-2af18970c072', Type='HDD', Model='ATA HGST HUS728T8TAL', Size='8001000000000', Firmware=''
13m         Warning   DriveReadyForRemoval           drive/2e183ae5-2497-424e-bf0e-603d4e134757                   Drive is ready for removal, Drive Details: SN='VDGZPDGD', Node='6c0c897d-26e1-4b33-95ba-2af18970c072', Type='HDD', Model='ATA HGST HUS728T8TAL', Size='8001000000000', Firmware=''
7m37s       Normal    Killing                        pod/web-3                                                    Stopping container nginx
7m24s       Normal    SuccessfulCreate               statefulset/web                                              create Pod web-3 in StatefulSet web successful
7m19s       Warning   DriveReadyForPhysicalRemoval   drive/2e183ae5-2497-424e-bf0e-603d4e134757                   Drive successfully removed from CSI, and ready for physical removal, Drive Details: SN='VDGZPDGD', Node='6c0c897d-26e1-4b33-95ba-2af18970c072', Type='HDD', Model='ATA HGST HUS728T8TAL', Size='8001000000000', Firmware=''
7m13s       Normal    DriveClean                     drive/2e183ae5-2497-424e-bf0e-603d4e134757                   Drive with path /dev/sdy, SN VDGZPDGD doesn't have filesystem, partition table, partitions or PV. Drive Details: SN='VDGZPDGD', Node='6c0c897d-26e1-4b33-95ba-2af18970c072', Type='HDD', Model='ATA HGST HUS728T8TAL', Size='8001000000000', Firmware=''
2m8s        Normal    DriveSuccessfullyRemoved       drive/2e183ae5-2497-424e-bf0e-603d4e134757                   Drive successfully removed. Drive status is: OFFLINE, previous status: ONLINE. Drive Details: SN='VDGZPDGD', Node='6c0c897d-26e1-4b33-95ba-2af18970c072', Type='HDD', Model='ATA HGST HUS728T8TAL', Size='8001000000000', Firmware=''
2m8s        Warning   DriveHealthUnknown             drive/2e183ae5-2497-424e-bf0e-603d4e134757                   Drive health is: UNKNOWN, previous state: SUSPECT. Drive Details: SN='VDGZPDGD', Node='6c0c897d-26e1-4b33-95ba-2af18970c072', Type='HDD', Model='ATA HGST HUS728T8TAL', Size='8001000000000', Firmware=''
2m8s        Warning   DriveHealthOverridden          drive/2e183ae5-2497-424e-bf0e-603d4e134757                   Drive health is overridden with: UNKNOWN, real state: SUSPECT. Drive Details: SN='VDGZPDGD', Node='6c0c897d-26e1-4b33-95ba-2af18970c072', Type='HDD', Model='ATA HGST HUS728T8TAL', Size='8001000000000', Firmware=''
```

codecov[bot]: # [Codecov](https://codecov.io/gh/dell/csi-baremetal/pull/487?src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=dell) Report
> Merging [#487](https://codecov.io/gh/dell/csi-baremetal/pull/487?src=pr&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=dell) (0da6a3c) into [master](https://codecov.io/gh/dell/csi-baremetal/commit/740c92d9d6a2879f20029b70e1763b2758ba9973?el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=dell) (740c92d) will **decrease** coverage by `0.02%`.
> The diff coverage is `60.00%`.

[![Impacted file tree graph](https://codecov.io/gh/dell/csi-baremetal/pull/487/graphs/tree.svg?width=650&height=150&src=pr&token=4D8WVNWX5O&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=dell)](https://codecov.io/gh/dell/csi-baremetal/pull/487?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=dell)

```diff
@@            Coverage Diff             @@
##           master     #487      +/-   ##
==========================================
- Coverage   59.13%   59.10%   -0.03%     
==========================================
  Files          60       60              
  Lines        6003     6006       +3     
==========================================
  Hits         3550     3550              
- Misses       2246     2248       +2     
- Partials      207      208       +1     
```

| Flag | Coverage Δ | |
|---|---|---|
| unittests | `59.10% <60.00%> (-0.03%)` | :arrow_down: |

Flags with carried forward coverage won't be shown. [Click here](https://docs.codecov.io/docs/carryforward-flags?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=dell#carryforward-flags-in-the-pull-request-comment) to find out more.

| [Impacted Files](https://codecov.io/gh/dell/csi-baremetal/pull/487?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=dell) | Coverage Δ | |
|---|---|---|
| [pkg/node/volumemgr.go](https://codecov.io/gh/dell/csi-baremetal/pull/487/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=dell#diff-cGtnL25vZGUvdm9sdW1lbWdyLmdv) | `69.69% <60.00%> (-0.22%)` | :arrow_down: |
| [pkg/scheduler/extender/extender.go](https://codecov.io/gh/dell/csi-baremetal/pull/487/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=dell#diff-cGtnL3NjaGVkdWxlci9leHRlbmRlci9leHRlbmRlci5nbw==) | `36.01% <0.00%> (-0.12%)` | :arrow_down: |
| [pkg/common/volume\_operations.go](https://codecov.io/gh/dell/csi-baremetal/pull/487/diff?src=pr&el=tree&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=dell#diff-cGtnL2NvbW1vbi92b2x1bWVfb3BlcmF0aW9ucy5nbw==) | `0.00% <0.00%> (ø)` | |

------

[Continue to review full report at Codecov](https://codecov.io/gh/dell/csi-baremetal/pull/487?src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=dell).
> **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=dell)
> `Δ = absolute <relative> (impact)`, `ø = not affected`, `? = missing data`
> Powered by [Codecov](https://codecov.io/gh/dell/csi-baremetal/pull/487?src=pr&el=footer&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=dell). Last update [740c92d...0da6a3c](https://codecov.io/gh/dell/csi-baremetal/pull/487?src=pr&el=lastupdated&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=dell). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=dell).


Patch:
diff --git a/api/v1/constants.go b/api/v1/constants.go
index 02f8f09..ef772b0 100644
--- a/api/v1/constants.go
+++ b/api/v1/constants.go
@@ -65,9 +65,11 @@ const (
 	DriveTypeNVMe = "NVME"
 
 	// Drive annotations
-	DriveAnnotationReplacement        = "replacement"
-	DriveAnnotationReplacementReady   = "ready"
+	DriveAnnotationRemoval            = "removal"
+	DriveAnnotationRemovalReady       = "ready"
 	DriveAnnotationVolumeStatusPrefix = "status"
+	// Deprecated annotations
+	DriveAnnotationReplacement        = "replacement"
 
 	// Volume operational status
 	OperationalStatusOperative   = "OPERATIVE"
diff --git a/docs/proposals/drive-replacement.md b/docs/proposals/drive-replacement.md
index e88253c..bcceedf 100644
--- a/docs/proposals/drive-replacement.md
+++ b/docs/proposals/drive-replacement.md
@@ -57,7 +57,7 @@ Negotiation between CSI and Operator is optional and will be done though annotat
   - `volumehealth.csi-baremetal/health: good/unknown/suspect/bad` - health of the underlying drive(s) 
   - `volumerelease.csi-baremetal/process: start|pause|stop` - inform to start|pause|stop volume release process
   - `volumerelease.csi-baremetal/release: processing` - system is working on data recovery/graceful IO shutdown
-  - `volumerelease.csi-baremetal/release: completed` - volume is released
+  - `volumerelease.csi-baremetal/release: done` - volume is released
   - `volumerelease.csi-baremetal/release: failed` - system failed to release volume
   - `volumerelease.csi-baremetal/recovery: [0:100]` - percent of recovery progress
   - `volumerelease.csi-baremetal/status: <status description>` - extra information. can be used to provide description for the release process. For example, *release=failed, recovery=50, status="recovery failed by timeout"*
@@ -81,7 +81,7 @@ To trigger physical drive replacement user must put the following annotation on
   - CSI sets drive operational status to `FAILED`
 * When drive operation status is `RELEASED` user can initiate physical drive replacement by setting `driveremove.csi-baremetal/replacement: ready` annotation on drive CR
   - CSI sets drive operational status to `REMOVING`
-  - Operator deletes PV(s)
+  - Operator deletes PVC(s)
   - CSI prepares drive for safe removal and starts LED locate
   - CSI sets drive operational status to `REMOVED` if all operations are passed successfully and `FAILED` otherwise
 * When drive operation status is `REMOVED`
diff --git a/pkg/base/linuxutils/lvm/lvm.go b/pkg/base/linuxutils/lvm/lvm.go
index 9fcc1f8..677eb0b 100644
--- a/pkg/base/linuxutils/lvm/lvm.go
+++ b/pkg/base/linuxutils/lvm/lvm.go
@@ -21,6 +21,7 @@ package lvm
 import (
 	"errors"
 	"fmt"
+	"regexp"
 	"strconv"
 	"strings"
 	"time"
@@ -28,6 +29,7 @@ import (
 	"github.com/sirupsen/logrus"
 
 	"github.com/dell/csi-baremetal/pkg/base/command"
+	errTypes "github.com/dell/csi-baremetal/pkg/base/error"
 	"github.com/dell/csi-baremetal/pkg/base/util"
 )
 
@@ -46,6 +48,10 @@ const (
 	PVsListCmdTmpl = lvmPath + "pvdisplay --short"
 	// VGCreateCmdTmpl create VG on provided PVs cmd
 	VGCreateCmdTmpl = lvmPath + "vgcreate --yes %s %s" // add VG name and PV names
+	// VGScanCmdTmpl searches for all VGs
+	VGScanCmdTmpl = lvmPath + "vgscan"
+	// VGRefreshCmdTmpl reactivates an LV using the latest metadata
+	VGRefreshCmdTmpl = lvmPath + "vgchange --refresh %s"
 	// VGRemoveCmdTmpl remove VG cmd
 	VGRemoveCmdTmpl = lvmPath + "vgremove --yes %s" // add VG name
 	// AllPVsCmd returns all physical volumes on the system
@@ -71,6 +77,8 @@ type WrapLVM interface {
 	PVCreate(dev string) error
 	PVRemove(name string) error
 	VGCreate(name string, pvs ...string) error
+	VGScan(name string) (bool, error)
+	VGReactivate(name string) error
 	VGRemove(name string) error
 	LVCreate(name, size, vgName string) error
 	LVRemove(fullLVName string) error
@@ -150,6 +158,55 @@ func (l *LVM) VGCreate(name string, pvs ...string) error {
 	return err
 }
 
+// VGScan scans for all VGs and checks for IO errors for specific volume group name
+// Receives name of VG name to scan and check
+// Return boolean (false if no IO errors detected, true otherwise) and error if command failed to execute
+func (l *LVM) VGScan(name string) (bool, error) {
+	// scan and check for VG errors
+	var (
+		stdout  string
+		stderr  string
+		err     error
+		exp     *regexp.Regexp
+		ioError = "input/output error"
+	)
+	// do the scan
+	if stdout, stderr, err = l.e.RunCmd(VGScanCmdTmpl, command.UseMetrics(true),
+		command.CmdName(strings.TrimSpace(VGScanCmdTmpl))); err != nil {
+		return false, err
+	}
+	// empty stdout is not expected. It must also contain VG name
+	if stdout == "" || !strings.Contains(stdout, name) {
+		return false, errTypes.ErrorNotFound
+	}
+	// find target volume group and check for IO errors
+	if exp, err = regexp.Compile(".*" + name + ".*\n*"); err != nil {
+		return false, err
+	}
+	lines := exp.FindAllString(stderr, -1)
+	for _, line := range lines {
+		if strings.Contains(strings.ToLower(line), ioError) {
+			return true, nil
+		}
+	}
+
+	return false, nil
+}
+
+// VGReactivate inactivates, scans and activates volume group to recover from disk missing scenario
+// Receives name of VG to re-activate
+// Returns error if something went wrong
+func (l *LVM) VGReactivate(name string) error {
+	l.log.Infof("Trying to re-activate volume group %s", name)
+	// re-activate related LVs
+	if _, _, err := l.e.RunCmd(fmt.Sprintf(VGRefreshCmdTmpl, name), command.UseMetrics(true),
+		command.CmdName(strings.TrimSpace(fmt.Sprintf(VGRefreshCmdTmpl, "")))); err != nil {
+		return err
+	}
+
+	return nil
+}
+
 // VGRemove removes volume group, ignore error if VG doesn't exist
 // Receives name of VG to remove
 // Returns error if something went wrong
diff --git a/pkg/crcontrollers/drive/drivecontroller.go b/pkg/crcontrollers/drive/drivecontroller.go
index adab0db..3b946e1 100644
--- a/pkg/crcontrollers/drive/drivecontroller.go
+++ b/pkg/crcontrollers/drive/drivecontroller.go
@@ -166,8 +166,8 @@ func (c *Controller) handleDriveUpdate(ctx context.Context, log *logrus.Entry, d
 		}
 		if allFound {
 			drive.Spec.Usage = apiV1.DriveUsageReleased
-			eventMsg := fmt.Sprintf("Drive is ready for replacement, %s", drive.GetDriveDescription())
-			c.eventRecorder.Eventf(drive, eventing.NormalType, eventing.DriveReadyForReplacement, eventMsg)
+			eventMsg := fmt.Sprintf("Drive is ready for removal, %s", drive.GetDriveDescription())
+			c.eventRecorder.Eventf(drive, eventing.WarningType, eventing.DriveReadyForRemoval, eventMsg)
 			toUpdate = true
 		}
 
@@ -177,8 +177,8 @@ func (c *Controller) handleDriveUpdate(ctx context.Context, log *logrus.Entry, d
 			break
 		}
 
-		status, found := drive.Annotations[apiV1.DriveAnnotationReplacement]
-		if !found || status != apiV1.DriveAnnotationReplacementReady {
+		status, found := getDriveAnnotationRemoval(drive.Annotations)
+		if !found || status != apiV1.DriveAnnotationRemovalReady {
 			break
 		}
 		toUpdate = true
@@ -190,10 +190,11 @@ func (c *Controller) handleDriveUpdate(ctx context.Context, log *logrus.Entry, d
 			return ignore, err
 		}
 		for _, vol := range volumes {
-			value, found := vol.Annotations[apiV1.DriveAnnotationReplacement]
-			if !found || value != apiV1.DriveAnnotationReplacementReady {
+			value, found := getDriveAnnotationRemoval(vol.Annotations)
+			if !found || value != apiV1.DriveAnnotationRemovalReady {
 				// need to update volume annotations
-				vol.Annotations[apiV1.DriveAnnotationReplacement] = apiV1.DriveAnnotationReplacementReady
+				vol.Annotations[apiV1.DriveAnnotationRemoval] = apiV1.DriveAnnotationRemovalReady
+				vol.Annotations[apiV1.DriveAnnotationReplacement] = apiV1.DriveAnnotationRemovalReady
 				if err := c.client.UpdateCR(ctx, vol); err != nil {
 					log.Errorf("Failed to update volume %s annotations, error: %v", vol.Name, err)
 					return ignore, err
@@ -214,11 +215,11 @@ func (c *Controller) handleDriveUpdate(ctx context.Context, log *logrus.Entry, d
 				drive.Spec.Usage = apiV1.DriveUsageFailed
 				// send error level alert
 				eventMsg := fmt.Sprintf("Failed to locale LED, %s", drive.GetDriveDescription())
-				c.eventRecorder.Eventf(drive, eventing.ErrorType, eventing.DriveReplacementFailed, eventMsg)
+				c.eventRecorder.Eventf(drive, eventing.ErrorType, eventing.DriveRemovalFailed, eventMsg)
 			} else {
-				// send info level alert
-				eventMsg := fmt.Sprintf("Drive successfully replaced, %s", drive.GetDriveDescription())
-				c.eventRecorder.Eventf(drive, eventing.NormalType, eventing.DriveSuccessfullyReplaced, eventMsg)
+				// send warning level alert (warning for attention), good level closes issue, need only send message
+				eventMsg := fmt.Sprintf("Drive successfully removed from CSI, and ready for physical removal, %s", drive.GetDriveDescription())
+				c.eventRecorder.Eventf(drive, eventing.WarningType, eventing.DriveReadyForPhysicalRemoval, eventMsg)
 			}
 			toUpdate = true
 		}
@@ -239,6 +240,15 @@ func (c *Controller) handleDriveUpdate(ctx context.Context, log *logrus.Entry, d
 	return ignore, nil
 }
 
+// For support deprecated Replacement annotation
+func getDriveAnnotationRemoval(annotations map[string]string) (string, bool) {
+	status, found := annotations[apiV1.DriveAnnotationRemoval]
+	if !found {
+		status, found = annotations[apiV1.DriveAnnotationReplacement]
+	}
+	return status, found
+}
+
 func (c *Controller) handleDriveStatus(ctx context.Context, drive *drivecrd.Drive) error {
 	volumes, err := c.crHelper.GetVolumesByLocation(ctx, drive.Spec.UUID)
 	if err != nil {
diff --git a/pkg/eventing/eventing.go b/pkg/eventing/eventing.go
index 91cf8f9..8ffa35a 100644
--- a/pkg/eventing/eventing.go
+++ b/pkg/eventing/eventing.go
@@ -32,20 +32,26 @@ const (
 	VolumeGoodHealth    = "VolumeGoodHealth"
 	VolumeSuspectHealth = "VolumeSuspectHealth"
 
-	DriveDiscovered           = "DriveDiscovered"
-	DriveHealthSuspect        = "DriveHealthSuspect"
-	DriveHealthFailure        = "DriveHealthFailure"
-	DriveHealthGood           = "DriveHealthGood"
-	DriveHealthUnknown        = "DriveHealthUnknown"
-	DriveStatusOnline         = "DriveStatusOnline"
-	DriveStatusOffline        = "DriveStatusOffline"
-	DriveReplacementFailed    = "DriveReplacementFailed"
-	DriveReadyForReplacement  = "DriveReadyForReplacement"
-	DriveSuccessfullyReplaced = "DriveSuccessfullyReplaced"
-	DriveHasData              = "DriveHasData"
-	DriveClean                = "DriveClean"
-	DriveHealthOverridden     = "DriveHealthOverridden"
+	DriveDiscovered              = "DriveDiscovered"
+	DriveHealthSuspect           = "DriveHealthSuspect"
+	DriveHealthFailure           = "DriveHealthFailure"
+	DriveHealthGood              = "DriveHealthGood"
+	DriveHealthUnknown           = "DriveHealthUnknown"
+	DriveStatusOnline            = "DriveStatusOnline"
+	DriveStatusOffline           = "DriveStatusOffline"
+	DriveRemovalFailed           = "DriveRemovalFailed"
+	DriveReadyForRemoval         = "DriveReadyForRemoval"
+	DriveReadyForPhysicalRemoval = "DriveReadyForPhysicalRemoval"
+	DriveSuccessfullyRemoved     = "DriveSuccessfullyRemoved"
+	DriveHasData                 = "DriveHasData"
+	DriveClean                   = "DriveClean"
+	DriveHealthOverridden        = "DriveHealthOverridden"
 
 	FakeAttachInvolved = "FakeAttachInvolved"
 	FakeAttachCleared  = "FakeAttachCleared"
+
+	VolumeGroupScanFailed         = "VolumeGroupScanFailed"
+	VolumeGroupScanInvolved       = "VolumeGroupScanInvolved"
+	VolumeGroupReactivateInvolved = "VolumeGroupReactivateInvolved"
+	VolumeGroupReactivateFailed   = "VolumeGroupReactivateFailed"
 )
diff --git a/pkg/mocks/linuxutils/lvm.go b/pkg/mocks/linuxutils/lvm.go
index fd487dd..53bfad6 100644
--- a/pkg/mocks/linuxutils/lvm.go
+++ b/pkg/mocks/linuxutils/lvm.go
@@ -53,6 +53,20 @@ func (m *MockWrapLVM) VGCreate(name string, pvs ...string) error {
 	return args.Error(0)
 }
 
+// VGScan is a mock implementation
+func (m *MockWrapLVM) VGScan(name string) (bool, error) {
+	args := m.Mock.Called(name)
+
+	return args.Bool(0), args.Error(0)
+}
+
+// VGReactivate is a mock implementation
+func (m *MockWrapLVM) VGReactivate(name string) error {
+	args := m.Mock.Called(name)
+
+	return args.Error(0)
+}
+
 // VGRemove is a mock implementations
 func (m *MockWrapLVM) VGRemove(name string) error {
 	args := m.Mock.Called(name)
diff --git a/pkg/node/volumemgr.go b/pkg/node/volumemgr.go
index 0382ff6..afc6936 100644
--- a/pkg/node/volumemgr.go
+++ b/pkg/node/volumemgr.go
@@ -485,7 +485,7 @@ func (m *VolumeManager) handleRemovingStatus(ctx context.Context, volume *volume
 					drive.Name, drive.Spec.Usage, err)
 				return ctrl.Result{Requeue: true}, err
 			}
-			m.sendEventForDrive(drive, eventing.ErrorType, eventing.DriveReplacementFailed, deleteVolumeFailedMsg, volume.Name, err)
+			m.sendEventForDrive(drive, eventing.ErrorType, eventing.DriveRemovalFailed, deleteVolumeFailedMsg, volume.Name, err)
 		}
 	} else {
 		ll.Infof("Volume - %s was successfully removed. Set status to Removed", volume.Spec.Id)
@@ -698,7 +698,7 @@ func (m *VolumeManager) updateDrivesCRs(ctx context.Context, drivesFromMgr []*ap
 
 func (m *VolumeManager) handleDriveUpdates(ctx context.Context, updates *driveUpdates) {
 	for _, updDrive := range updates.Updated {
-		m.handleDriveStatusChange(ctx, &updDrive.CurrentState.Spec)
+		m.handleDriveStatusChange(ctx, updDrive)
 	}
 	m.createEventsForDriveUpdates(updates)
 }
@@ -910,22 +910,46 @@ func (m *VolumeManager) getProvisionerForVolume(vol *api.Volume) p.Provisioner {
 // handleDriveStatusChange removes AC that is based on unhealthy drive, returns AC if drive returned to healthy state,
 // mark volumes of the unhealthy drive as unhealthy.
 // Receives golang context and api.Drive that should be handled
-func (m *VolumeManager) handleDriveStatusChange(ctx context.Context, drive *api.Drive) {
+func (m *VolumeManager) handleDriveStatusChange(ctx context.Context, drive updatedDrive) {
+	cur := drive.CurrentState.Spec
+	prev := drive.PreviousState.Spec
 	ll := m.log.WithFields(logrus.Fields{
 		"method":  "handleDriveStatusChange",
-		"driveID": drive.UUID,
+		"driveID": cur.UUID,
 	})
 
-	ll.Infof("The new drive status from DriveMgr is %s", drive.Health)
+	ll.Infof("The new cur status from DriveMgr is %s", cur.Health)
 
 	// Handle resources without LogicalVolumeGroup
 	// Remove AC based on disk with health BAD, SUSPECT, UNKNOWN
-	lvg, err := m.cachedCrHelper.GetLVGByDrive(ctx, drive.UUID)
+	lvg, err := m.cachedCrHelper.GetLVGByDrive(ctx, cur.UUID)
 	if lvg != nil {
+		name := lvg.Name
 		// TODO handle situation when LVG health is changing from Bad/Suspect to Good https://github.com/dell/csi-baremetal/issues/385
-		lvg.Spec.Health = drive.Health
+		lvg.Spec.Health = cur.Health
 		if err := m.k8sClient.UpdateCR(ctx, lvg); err != nil {
-			ll.Errorf("Failed to update lvg CR's %s health status: %v", lvg.Name, err)
+			ll.Errorf("Failed to update lvg CR's %s health status: %v", name, err)
+		}
+		// check for missing disk and re-activate volume group if needed
+		if prev.Status == apiV1.DriveStatusOffline && cur.Status == apiV1.DriveStatusOnline {
+			ll.Infof("Scan volume group %s for IO errors", name)
+			m.recorder.Eventf(lvg, eventing.NormalType, eventing.VolumeGroupScanInvolved, "Check for IO errors")
+			if ok, err := m.lvmOps.VGScan(name); err != nil { //nolint:gocritic
+				ll.Errorf("Failed to scan volume group %s for IO errors: %v", name, err)
+				m.recorder.Eventf(lvg, eventing.ErrorType, eventing.VolumeGroupScanFailed, err.Error())
+			} else if ok {
+				// IO errors detected. Need to re-activate volume group
+				ll.Errorf("IO errors detected for volume group %s", name)
+				m.recorder.Eventf(lvg, eventing.WarningType, eventing.VolumeGroupReactivateInvolved,
+					"IO errors detected")
+				if err := m.lvmOps.VGReactivate(name); err != nil {
+					// need to send an event if operation failed
+					ll.Errorf("Failed to re-activate volume group %s: %v", name, err)
+					m.recorder.Eventf(lvg, eventing.ErrorType, eventing.VolumeGroupReactivateFailed, err.Error())
+				}
+			} else {
+				ll.Infof("No IO errors detected for volume group %s", name)
+			}
 		}
 	} else {
 		errMsg := "Failed get LogicalVolumeGroup CR"
@@ -935,12 +959,12 @@ func (m *VolumeManager) handleDriveStatusChange(ctx context.Context, drive *api.
 		ll.Errorf(errMsg)
 	}
 	// Set disk's health status to volume CR
-	volumes, _ := m.cachedCrHelper.GetVolumesByLocation(ctx, drive.UUID)
+	volumes, _ := m.cachedCrHelper.GetVolumesByLocation(ctx, cur.UUID)
 	for _, vol := range volumes {
-		ll.Infof("Setting updated status %s to volume %s", drive.Health, vol.Name)
+		ll.Infof("Setting updated status %s to volume %s", cur.Health, vol.Name)
 		// save previous health state
 		prevHealthState := vol.Spec.Health
-		vol.Spec.Health = drive.Health
+		vol.Spec.Health = cur.Health
 		// initiate volume release
 		// TODO need to check for specific annotation instead
 		if vol.Spec.Health == apiV1.HealthBad || vol.Spec.Health == apiV1.HealthSuspect {
@@ -956,7 +980,7 @@ func (m *VolumeManager) handleDriveStatusChange(ctx context.Context, drive *api.
 		if vol.Spec.Health == apiV1.HealthBad {
 			m.recorder.Eventf(vol, eventing.WarningType, eventing.VolumeBadHealth,
 				"Volume health transitioned from %s to %s. Inherited from %s drive on %s)",
-				prevHealthState, vol.Spec.Health, drive.Health, drive.NodeId)
+				prevHealthState, vol.Spec.Health, cur.Health, cur.NodeId)
 		}
 	}
 	// Handle resources with LogicalVolumeGroup
@@ -1029,8 +1053,13 @@ func (m *VolumeManager) createEventForDriveStatusChange(
 	case apiV1.DriveStatusOnline:
 		reason = eventing.DriveStatusOnline
 	case apiV1.DriveStatusOffline:
-		eventType = eventing.ErrorType
-		reason = eventing.DriveStatusOffline
+		if drive.Spec.Usage == apiV1.DriveUsageRemoved {
+			reason = eventing.DriveSuccessfullyRemoved
+			statusMsgTemplate = "Drive successfully removed. " + statusMsgTemplate
+		} else {
+			eventType = eventing.ErrorType
+			reason = eventing.DriveStatusOffline
+		}
 	default:
 		return
 	}
@@ -1050,7 +1079,7 @@ func (m *VolumeManager) createEventForDriveHealthOverridden(
 
 func (m *VolumeManager) sendEventForDrive(drive *drivecrd.Drive, eventtype, reason, messageFmt string,
 	args ...interface{}) {
-	messageFmt += drive.GetDriveDescription()
+	messageFmt += " " + drive.GetDriveDescription()
 	m.recorder.Eventf(drive, eventtype, reason, messageFmt, args...)
 }
 


Test patch:
diff --git a/pkg/base/linuxutils/lvm/lvm_test.go b/pkg/base/linuxutils/lvm/lvm_test.go
index 197ef30..aae64fa 100644
--- a/pkg/base/linuxutils/lvm/lvm_test.go
+++ b/pkg/base/linuxutils/lvm/lvm_test.go
@@ -19,6 +19,7 @@ package lvm
 import (
 	"errors"
 	"fmt"
+	errTypes "github.com/dell/csi-baremetal/pkg/base/error"
 	"strings"
 	"testing"
 
@@ -95,6 +96,57 @@ func TestLinuxUtils_VGCreate(t *testing.T) {
 	err = l.VGCreate(vg, dev1, dev2)
 	assert.Equal(t, expectedErr, err)
 }
+func TestLinuxUtils_VGScan(t *testing.T) {
+	var (
+		e           = &mocks.GoMockExecutor{}
+		l           = NewLVM(e, testLogger)
+		vg          = "test-vg"
+		cmd         = VGScanCmdTmpl
+		ok 			bool
+		err         error
+		expectedErr = errors.New("error")
+	)
+
+	// error not found
+	e.OnCommand(cmd).Return("", "", nil).Times(1)
+	ok, err = l.VGScan(vg)
+	assert.Equal(t, err, errTypes.ErrorNotFound)
+	assert.False(t, ok)
+
+	// error - expected
+	e.OnCommand(cmd).Return("", "", expectedErr).Times(1)
+	ok, err = l.VGScan(vg)
+	assert.False(t, ok)
+	assert.Equal(t, err, expectedErr)
+
+	// IO error detected
+	e.OnCommand(cmd).Return("Found volume group \"" + vg + "\" using metadata type lvm2",
+		"/dev/" + vg + "/test-lv: Input/output error", nil).Times(1)
+	ok, err = l.VGScan(vg)
+	assert.True(t, ok)
+	assert.Nil(t, err)
+
+	// IO error not detected - multiple lines
+	e.OnCommand(cmd).Return("Found volume group \"" + vg + "\" using metadata type lvm2",
+		"/dev/%s/test-lv: no errors\n/dev/other-vg/test-lv: Input/output error", nil).Times(1)
+	ok, err = l.VGScan(vg)
+	assert.False(t, ok)
+	assert.Nil(t, err)
+
+	// IO error detected - multiple lines
+	e.OnCommand(cmd).Return("Found volume group \"" + vg + "\" using metadata type lvm2",
+		"/dev/" + vg + "/test-lv: no errors\n/dev/" + vg + "/test-lv-2: Input/output error", nil).Times(1)
+	ok, err = l.VGScan(vg)
+	assert.True(t, ok)
+	assert.Nil(t, err)
+
+	// error - wrong volume group name
+	incorrectName := "*"
+	e.OnCommand(cmd).Return(incorrectName, "", nil).Times(1)
+	ok, err = l.VGScan(incorrectName)
+	assert.False(t, ok)
+	assert.NotNil(t, err)
+}
 
 func TestLinuxUtils_VGRemove(t *testing.T) {
 	var (
diff --git a/pkg/node/volumemgr_test.go b/pkg/node/volumemgr_test.go
index 09a9742..3d59519 100644
--- a/pkg/node/volumemgr_test.go
+++ b/pkg/node/volumemgr_test.go
@@ -808,16 +808,22 @@ func TestVolumeManager_handleDriveStatusChange(t *testing.T) {
 	drive := drive1
 	drive.UUID = driveUUID
 	drive.Health = apiV1.HealthBad
+	driveCR := &drivecrd.Drive{Spec:drive}
+
+	update := updatedDrive{
+		PreviousState: driveCR,
+		CurrentState:  driveCR,
+	}
 
 	// Check AC deletion
-	vm.handleDriveStatusChange(testCtx, &drive)
+	vm.handleDriveStatusChange(testCtx, update)
 	vol := volCR
 	vol.Spec.Location = driveUUID
 	err = vm.k8sClient.CreateCR(testCtx, testID, &vol)
 	assert.Nil(t, err)
 
 	// Check volume's health change
-	vm.handleDriveStatusChange(testCtx, &drive)
+	vm.handleDriveStatusChange(testCtx, update)
 	rVolume := &vcrd.Volume{}
 	err = vm.k8sClient.ReadCR(testCtx, testID, volCR.Namespace, rVolume)
 	assert.Nil(t, err)
@@ -828,7 +834,7 @@ func TestVolumeManager_handleDriveStatusChange(t *testing.T) {
 	err = vm.k8sClient.CreateCR(testCtx, testLVGName, &lvg)
 	assert.Nil(t, err)
 	// Check lvg's health change
-	vm.handleDriveStatusChange(testCtx, &drive)
+	vm.handleDriveStatusChange(testCtx, update)
 	updatedLVG := &lvgcrd.LogicalVolumeGroup{}
 	err = vm.k8sClient.ReadCR(testCtx, testLVGName, "", updatedLVG)
 	assert.Nil(t, err)
@@ -1010,6 +1016,23 @@ func TestVolumeManager_createEventsForDriveUpdates(t *testing.T) {
 		assert.True(t, expectEvent(drive1CR, eventing.ErrorType, eventing.DriveStatusOffline))
 		assert.True(t, expectEvent(drive1CR, eventing.WarningType, eventing.DriveHealthUnknown))
 	})
+
+	t.Run("Drive removed", func(t *testing.T) {
+		init()
+		modifiedDrive := drive1CR.DeepCopy()
+		modifiedDrive.Spec.Usage = apiV1.DriveUsageRemoved
+		modifiedDrive.Spec.Status = apiV1.DriveStatusOffline
+		modifiedDrive.Spec.Health = apiV1.HealthUnknown
+
+		upd := &driveUpdates{
+			Updated: []updatedDrive{{
+				PreviousState: drive1CR,
+				CurrentState:  modifiedDrive}},
+		}
+		mgr.createEventsForDriveUpdates(upd)
+		assert.True(t, expectEvent(drive1CR, eventing.NormalType, eventing.DriveSuccessfullyRemoved))
+		assert.True(t, expectEvent(drive1CR, eventing.WarningType, eventing.DriveHealthUnknown))
+	})
 }
 
 func TestVolumeManager_isShouldBeReconciled(t *testing.T) {