Repository: longhorn/longhorn-manager
Pull request #3694: feat: orphaned runtime instance cleanup
Author: login: COLDTURNIP
Created: 2025-04-17T10:09:00Z | Merged: 2025-04-24T08:51:16Z

#### Which issue(s) this PR fixes:
<!--
Use `Issue #<issue number>` or `Issue longhorn/longhorn#<issue number>` or `Issue (paste link of issue)`. DON'T use `Fixes #<issue number>` or `Fixes (paste link of issue)`, as it will automatically close the linked issue when the PR is merged.
-->
Issue longhorn/longhorn#6764

#### What this PR does / why we need it:

Implement orphaned runtime instance cleanup.
LEP: longhorn/longhorn#10675

#### Special notes for your reviewer:

#### Additional documentation or context



Diff:
diff --git a/api/model.go b/api/model.go
index 83b3c80b94..75a9f997a0 100644
--- a/api/model.go
+++ b/api/model.go
@@ -2400,6 +2400,7 @@ func toOrphanResource(orphan *longhorn.Orphan) *Orphan {
 		OrphanSpec: longhorn.OrphanSpec{
 			NodeID:     orphan.Spec.NodeID,
 			Type:       orphan.Spec.Type,
+			DataEngine: orphan.Spec.DataEngine,
 			Parameters: orphan.Spec.Parameters,
 		},
 	}
diff --git a/controller/instance_manager_controller.go b/controller/instance_manager_controller.go
index 26e50f9538..58a9f3e202 100644
--- a/controller/instance_manager_controller.go
+++ b/controller/instance_manager_controller.go
@@ -87,6 +87,8 @@ type InstanceManagerMonitor struct {
 	proxyClient engineapi.EngineClientProxy
 }
 
+type instanceProcessMap map[string]longhorn.InstanceProcess
+
 func updateInstanceManagerVersion(im *longhorn.InstanceManager) error {
 	cli, err := engineapi.NewInstanceManagerClient(im, false)
 	if err != nil {
@@ -170,6 +172,13 @@ func NewInstanceManagerController(
 	}
 	imc.cacheSyncs = append(imc.cacheSyncs, ds.KubeNodeInformer.HasSynced)
 
+	if _, err = ds.OrphanInformer.AddEventHandlerWithResyncPeriod(cache.ResourceEventHandlerFuncs{
+		AddFunc: imc.enqueueInstanceManagerOrphan,
+	}, 0); err != nil {
+		return nil, err
+	}
+	imc.cacheSyncs = append(imc.cacheSyncs, ds.OrphanInformer.HasSynced)
+
 	if _, err = ds.SettingInformer.AddEventHandlerWithResyncPeriod(
 		cache.FilteringResourceEventHandler{
 			FilterFunc: imc.isResponsibleForSetting,
@@ -200,7 +209,8 @@ func (imc *InstanceManagerController) isResponsibleForSetting(obj interface{}) b
 	}
 
 	return types.SettingName(setting.Name) == types.SettingNameKubernetesClusterAutoscalerEnabled ||
-		types.SettingName(setting.Name) == types.SettingNameV2DataEngineCPUMask
+		types.SettingName(setting.Name) == types.SettingNameV2DataEngineCPUMask ||
+		types.SettingName(setting.Name) == types.SettingNameOrphanResourceAutoDeletion
 }
 
 func isInstanceManagerPod(obj interface{}) bool {
@@ -333,6 +343,7 @@ func (imc *InstanceManagerController) syncInstanceManager(key string) (err error
 
 	if im.DeletionTimestamp != nil {
 		log.Warnf("Deleting instance manager pod %v since the instance manager is being deleted", name)
+		// the orphan instance CRs will be also deleted because of owner reference
 		return imc.cleanupInstanceManagerPod(im.Name)
 	}
 
@@ -389,6 +400,10 @@ func (imc *InstanceManagerController) syncInstanceManager(key string) (err error
 		return err
 	}
 
+	if err := imc.syncOrphans(im); err != nil {
+		return err
+	}
+
 	return nil
 }
 
@@ -988,6 +1003,32 @@ func (imc *InstanceManagerController) deleteInstanceManagerPDB(im *longhorn.Inst
 	return nil
 }
 
+func (imc *InstanceManagerController) syncOrphans(im *longhorn.InstanceManager) error {
+	// Instances are live inside the instance manager pod.
+	// Remove the instance orphan CRs when the instance manager pod is deleted or not running.
+	var isInstanceManagerPodDeletedOrNotRunning bool
+	if im.Status.CurrentState != longhorn.InstanceManagerStateRunning {
+		isInstanceManagerPodDeletedOrNotRunning = true
+	} else {
+		switch pod, err := imc.ds.GetPodRO(im.Namespace, im.Name); {
+		case datastore.ErrorIsNotFound(err):
+			isInstanceManagerPodDeletedOrNotRunning = true
+		case err != nil:
+			return errors.Wrapf(err, "failed to check instance manager %v pod state while sync orphans", im.Name)
+		default:
+			isInstanceManagerPodDeletedOrNotRunning = pod == nil
+		}
+	}
+
+	// Once the original owner controller lost the ownership of the instance manager CR, it is no longer able to monitor the instance status.
+	// This may be caused by network outage or Longhorn manager failure.
+	// Remove the corresponding orphan CRs, and they will be resync after the owner instance manager gets back.
+	instanceManagerOwnershipChanged := im.Spec.NodeID != imc.controllerID
+
+	isInstanceManagerTerminating := isInstanceManagerPodDeletedOrNotRunning || instanceManagerOwnershipChanged
+	return imc.deleteOrphans(im, isInstanceManagerTerminating)
+}
+
 const ITERATE_NAME_LIMIT = 5
 
 func formatInstanceMessage(im *longhorn.InstanceManager) string {
@@ -1247,6 +1288,15 @@ func (imc *InstanceManagerController) enqueueKubernetesNode(obj interface{}) {
 	imc.enqueueInstanceManagersForNode(kubernetesNode.Name)
 }
 
+func (imc *InstanceManagerController) enqueueInstanceManagerOrphan(obj interface{}) {
+	orphan, ok := obj.(*longhorn.Orphan)
+	if !ok {
+		return
+	}
+
+	imc.enqueueInstanceManagersForNode(orphan.Spec.NodeID)
+}
+
 func (imc *InstanceManagerController) enqueueInstanceManagersForNode(nodeName string) {
 	node, err := imc.ds.GetNodeRO(nodeName)
 	if err != nil {
@@ -1610,6 +1660,123 @@ func (imc *InstanceManagerController) createInstanceManagerPodSpec(im *longhorn.
 	return podSpec, nil
 }
 
+func (imc *InstanceManagerController) deleteOrphans(im *longhorn.InstanceManager, isInstanceManagerTerminating bool) error {
+	autoDeletionTypes, err := imc.ds.GetSettingOrphanResourceAutoDeletion()
+	var autoDeleteEngine = false
+	var autoDeleteReplica = false
+	if err != nil {
+		imc.logger.WithError(err).Warnf("Failed to fetch orphan auto deletion setting, disabled by default")
+	} else {
+		autoDeleteEngine = autoDeletionTypes[types.OrphanResourceTypeEngineInstance]
+		autoDeleteReplica = autoDeletionTypes[types.OrphanResourceTypeReplicaInstance]
+	}
+
+	orphanList, err := imc.ds.ListOrphansForEngineAndReplicaInstancesRO(im.Spec.NodeID)
+	if err != nil {
+		return err
+	}
+	multiError := util.NewMultiError()
+	for _, orphan := range orphanList {
+		if !orphan.DeletionTimestamp.IsZero() {
+			continue
+		}
+
+		var isDeletable = false
+		switch orphan.Spec.Type {
+		case longhorn.OrphanTypeEngineInstance:
+			isDeletable, err = imc.isOrphanDeletable(orphan, isInstanceManagerTerminating || autoDeleteEngine, im.Status.InstanceEngines, imc.isEngineScheduledOnCurrentNode)
+		case longhorn.OrphanTypeReplicaInstance:
+			isDeletable, err = imc.isOrphanDeletable(orphan, isInstanceManagerTerminating || autoDeleteReplica, im.Status.InstanceReplicas, imc.isReplicaScheduledOnCurrentNode)
+		}
+		if err != nil {
+			multiError.Append(util.NewMultiError(fmt.Sprintf("%v: %v", orphan.Name, err)))
+		} else if isDeletable {
+			if err := imc.ds.DeleteOrphan(orphan.Name); err != nil && !apierrors.IsNotFound(err) {
+				multiError.Append(util.NewMultiError(fmt.Sprintf("%v: %v", orphan.Name, err)))
+			}
+		}
+	}
+	if len(multiError) > 0 {
+		return fmt.Errorf("instance manager failed to delete orphan CR: %v", multiError.Join())
+	}
+	return nil
+}
+
+func (imc *InstanceManagerController) isOrphanDeletable(orphan *longhorn.Orphan, autoDelete bool, instanceMap instanceProcessMap, isInstanceScheduledOnNode func(node, instance string) (bool, error)) (isDeletable bool, err error) {
+	if autoDelete {
+		return true, nil
+	}
+
+	instanceName := orphan.Spec.Parameters[longhorn.OrphanInstanceName]
+	instanceProc, instanceExist := instanceMap[instanceName]
+	instanceCRScheduledBack, err := isInstanceScheduledOnNode(orphan.Spec.NodeID, instanceName)
+	if err != nil {
+		return false, errors.Wrapf(err, "failed to check CR scheduled node for %v orphan %v", orphan.Spec.Type, instanceName)
+	}
+
+	var instanceState longhorn.InstanceState
+	if instanceExist {
+		instanceState = instanceProc.Status.State
+	} else {
+		instanceState = longhorn.InstanceStateTerminated
+	}
+
+	imc.logger.WithFields(logrus.Fields{
+		"instanceExist":           instanceExist,
+		"instanceState":           instanceState,
+		"instanceCRScheduledBack": instanceCRScheduledBack,
+	}).Debugf("Checked deletable for %s orphan instance %s", orphan.Spec.Type, orphan.Name)
+	if !instanceExist || instanceCRScheduledBack {
+		return true, nil
+	}
+	return false, nil
+}
+
+func (imc *InstanceManagerController) isEngineScheduledOnCurrentNode(node string, instance string) (bool, error) {
+	existEngine, err := imc.ds.GetEngineRO(instance)
+	switch {
+	case err == nil:
+		// Engine CR still exists - check the ownership
+		return imc.isInstanceScheduledOnCurrentNode(node, &existEngine.Spec.InstanceSpec, &existEngine.Status.InstanceStatus), nil
+	case apierrors.IsNotFound(err):
+		// Engine CR not found - instance is orphaned
+		return false, nil
+	default:
+		// Unexpected error - unable to check if engine instance is orphaned or not
+		return false, err
+	}
+}
+
+func (imc *InstanceManagerController) isReplicaScheduledOnCurrentNode(node string, instance string) (bool, error) {
+	existReplica, err := imc.ds.GetReplicaRO(instance)
+	switch {
+	case err == nil:
+		// Replica CR still exists - check the ownership
+		return imc.isInstanceScheduledOnCurrentNode(node, &existReplica.Spec.InstanceSpec, &existReplica.Status.InstanceStatus), nil
+	case apierrors.IsNotFound(err):
+		// Replica CR not found - instance is orphaned
+		return false, nil
+	default:
+		// Unexpected error - unable to check if replica instance is orphaned or not
+		return false, err
+	}
+}
+
+// isInstanceScheduledOnCurrentNode returns true only when it is very certain that an instance is scheduled on this node
+func (imc *InstanceManagerController) isInstanceScheduledOnCurrentNode(node string, spec *longhorn.InstanceSpec, status *longhorn.InstanceStatus) bool {
+	if status.CurrentState != spec.DesireState {
+		return false
+	}
+	switch status.CurrentState {
+	case longhorn.InstanceStateRunning:
+		return status.OwnerID == spec.NodeID && spec.NodeID == node
+	case longhorn.InstanceStateStopped:
+		return spec.NodeID == node
+	default:
+		return false
+	}
+}
+
 func (imc *InstanceManagerController) startBackingImageMonitoring(im *longhorn.InstanceManager) {
 	log := imc.logger.WithField("instance manager", im.Name)
 
@@ -1942,48 +2109,57 @@ func (m *InstanceManagerMonitor) Run() {
 			if !needUpdate {
 				continue
 			}
-			if needStop := m.pollAndUpdateInstanceMap(); needStop {
+			instanceManager, instanceMap, needStop := m.pollInstanceMap()
+			if needStop {
 				return
 			}
+			if instanceManager != nil && instanceMap != nil {
+				m.syncInstances(instanceManager, instanceMap)
+				m.syncOrphans(instanceManager, instanceMap)
+			}
 		case <-m.stopCh:
 			return
 		}
 	}
 }
 
-func (m *InstanceManagerMonitor) pollAndUpdateInstanceMap() (needStop bool) {
+func (m *InstanceManagerMonitor) pollInstanceMap() (im *longhorn.InstanceManager, instanceMap instanceProcessMap, needStop bool) {
 	im, err := m.ds.GetInstanceManager(m.Name)
 	if err != nil {
 		if datastore.ErrorIsNotFound(err) {
 			m.logger.Warn("Stop monitoring because the instance manager no longer exists")
-			return true
+			return nil, nil, true
 		}
 		utilruntime.HandleError(errors.Wrapf(err, "failed to get instance manager %v for monitoring", m.Name))
-		return false
+		return nil, nil, false
 	}
 
 	if im.Status.OwnerID != m.controllerID {
 		m.logger.Warnf("Stop monitoring the instance manager on this node (%v) because the instance manager has new ownerID %v", m.controllerID, im.Status.OwnerID)
-		return true
+		return nil, nil, true
 	}
 
 	resp, err := m.client.InstanceList()
 	if err != nil {
 		utilruntime.HandleError(errors.Wrapf(err, "failed to poll instance info to update instance manager %v", m.Name))
-		return false
+		return nil, nil, false
 	}
-	if !m.updateInstanceMap(im, resp) {
-		return false
+	return im, resp, false
+}
+
+func (m *InstanceManagerMonitor) syncInstances(im *longhorn.InstanceManager, instanceMap instanceProcessMap) {
+	if !m.updateInstanceMap(im, instanceMap) {
+		return
 	}
 	if _, err := m.ds.UpdateInstanceManagerStatus(im); err != nil {
 		utilruntime.HandleError(errors.Wrapf(err, "failed to update instance map for instance manager %v", m.Name))
-		return false
+		return
 	}
 
 	clusterAutoscalerEnabled, err := m.ds.GetSettingAsBool(types.SettingNameKubernetesClusterAutoscalerEnabled)
 	if err != nil {
 		utilruntime.HandleError(errors.Wrapf(err, "failed to get %v setting for instance manager %v", types.SettingNameKubernetesClusterAutoscalerEnabled, m.Name))
-		return false
+		return
 	}
 
 	// During volume attaching/detaching, it is likely both the engine and replica
@@ -1995,39 +2171,28 @@ func (m *InstanceManagerMonitor) pollAndUpdateInstanceMap() (needStop bool) {
 	if clusterAutoscalerEnabled && im.Spec.Type == longhorn.InstanceManagerTypeEngine {
 		m.nodeCallback(m.controllerID)
 	}
-
-	return false
 }
 
-func (m *InstanceManagerMonitor) updateInstanceMap(im *longhorn.InstanceManager, resp map[string]longhorn.InstanceProcess) bool {
+func (m *InstanceManagerMonitor) updateInstanceMap(im *longhorn.InstanceManager, instanceMap instanceProcessMap) bool {
 	switch {
 	case im.Status.APIVersion < 4:
-		if reflect.DeepEqual(im.Status.Instances, resp) { // nolint: staticcheck
+		if reflect.DeepEqual(im.Status.Instances, instanceMap) { // nolint: staticcheck
 			return false
 		}
 
-		im.Status.Instances = resp // nolint: staticcheck
+		im.Status.Instances = instanceMap // nolint: staticcheck
 	default:
-		engineProcess := map[string]longhorn.InstanceProcess{}
-		replicaProcess := map[string]longhorn.InstanceProcess{}
-		for name, process := range resp {
-			switch process.Status.Type {
-			case longhorn.InstanceTypeEngine:
-				engineProcess[name] = process
-			case longhorn.InstanceTypeReplica:
-				replicaProcess[name] = process
-			}
-		}
+		engineProcesses, replicaProcesses := m.categorizeProcesses(instanceMap)
 
 		// reflect.DeepEqual treats the two maps `var m1 map[string]process` and `m2 := map[string]process` as different maps.
 		// Therefore, to prevent unnecessary updates, we must check both that the length of the maps is zero and that the maps are identical.
-		if ((len(im.Status.InstanceEngines) == 0 && len(engineProcess) == 0) || reflect.DeepEqual(im.Status.InstanceEngines, engineProcess)) &&
-			((len(im.Status.InstanceReplicas) == 0 && len(replicaProcess) == 0) || reflect.DeepEqual(im.Status.InstanceReplicas, replicaProcess)) {
+		if ((len(im.Status.InstanceEngines) == 0 && len(engineProcesses) == 0) || reflect.DeepEqual(im.Status.InstanceEngines, engineProcesses)) &&
+			((len(im.Status.InstanceReplicas) == 0 && len(replicaProcesses) == 0) || reflect.DeepEqual(im.Status.InstanceReplicas, replicaProcesses)) {
 			return false
 		}
 
-		im.Status.InstanceEngines = engineProcess
-		im.Status.InstanceReplicas = replicaProcess
+		im.Status.InstanceEngines = engineProcesses
+		im.Status.InstanceReplicas = replicaProcesses
 	}
 	return true
 }
@@ -2044,6 +2209,127 @@ func (m *InstanceManagerMonitor) StopMonitorWithLock() {
 	m.done = true
 }
 
+func (m *InstanceManagerMonitor) syncOrphans(im *longhorn.InstanceManager, instanceMap instanceProcessMap) {
+	engineProcesses, replicaProcesses := m.categorizeProcesses(instanceMap)
+	existOrphansList, err := m.ds.ListOrphansForEngineAndReplicaInstancesRO(im.Spec.NodeID)
+	if err != nil {
+		m.logger.WithError(err).Errorf("Failed to list orphans on node %s", im.Spec.NodeID)
+		return
+	}
+	existOrphans := make(map[string]bool)
+	for _, orphan := range existOrphansList {
+		existOrphans[orphan.Name] = true
+	}
+
+	// exam instances and create orphan CRs
+	m.createOrphanForInstances(existOrphans, im, engineProcesses, longhorn.OrphanTypeEngineInstance, m.isEngineOrphaned)
+	m.createOrphanForInstances(existOrphans, im, replicaProcesses, longhorn.OrphanTypeReplicaInstance, m.isReplicaOrphaned)
+}
+
+func (m *InstanceManagerMonitor) isEngineOrphaned(instanceName, node string) (bool, error) {
+	existEngine, err := m.ds.GetEngineRO(instanceName)
+	switch {
+	case err == nil:
+		// Engine CR still exists - check the ownership
+		return m.isInstanceOrphanedOnNode(&existEngine.Spec.InstanceSpec, &existEngine.Status.InstanceStatus, node), nil
+	case apierrors.IsNotFound(err):
+		// Engine CR not found - instance is orphaned
+		return true, nil
+	default:
+		// Unexpected error - unable to check if engine instance is orphaned or not
+		return false, err
+	}
+}
+
+func (m *InstanceManagerMonitor) isReplicaOrphaned(instanceName, node string) (bool, error) {
+	existReplica, err := m.ds.GetReplicaRO(instanceName)
+	switch {
+	case err == nil:
+		// Replica CR still exists - check the ownership
+		return m.isInstanceOrphanedOnNode(&existReplica.Spec.InstanceSpec, &existReplica.Status.InstanceStatus, node), nil
+	case apierrors.IsNotFound(err):
+		// Replica CR not found - instance is orphaned
+		return true, nil
+	default:
+		// Unexpected error - unable to check if replica instance is orphaned or not
+		return false, err
+	}
+}
+
+// isInstanceOrphanedOnNode returns true only when it is very certain that an instance is scheduled on another node
+func (m *InstanceManagerMonitor) isInstanceOrphanedOnNode(spec *longhorn.InstanceSpec, status *longhorn.InstanceStatus, node string) bool {
+	if status.CurrentState != spec.DesireState {
+		return false
+	}
+	switch status.CurrentState {
+	case longhorn.InstanceStateRunning:
+		return status.OwnerID == spec.NodeID && spec.NodeID != node
+	case longhorn.InstanceStateStopped:
+		return spec.NodeID != node
+	default:
+		return false
+	}
+}
+
+func (m *InstanceManagerMonitor) createOrphanForInstances(existOrphans map[string]bool, im *longhorn.InstanceManager, instanceMap instanceProcessMap, orphanType longhorn.OrphanType, orphanFilter func(instanceName, node string) (bool, error)) {
+	for instanceName, instance := range instanceMap {
+		orphanName := types.GetOrphanChecksumNameForOrphanedInstance(instanceName, m.controllerID, string(instance.Spec.DataEngine))
+		if _, isExist := existOrphans[orphanName]; isExist {
+			continue
+		}
+		if isOrphaned, err := orphanFilter(instanceName, im.Spec.NodeID); err != nil {
+			m.logger.WithError(err).Errorf("Failed to check %v orphan for instance %v", orphanType, instanceName)
+		} else if isOrphaned {
+			m.logger.Infof("Instance %v is orphaned", instanceName)
+			newOrphan, err := m.createOrphan(orphanName, im, instanceName, orphanType, instance.Spec.DataEngine)
+			if err != nil {
+				m.logger.WithError(err).Errorf("Failed to create %v orphan for instance %v", orphanType, instanceName)
+			} else if newOrphan != nil {
+				existOrphans[newOrphan.Name] = true
+			}
+		}
+	}
+}
+
+func (m *InstanceManagerMonitor) createOrphan(name string, im *longhorn.InstanceManager, instanceName string, orphanType longhorn.OrphanType, dataEngineType longhorn.DataEngineType) (*longhorn.Orphan, error) {
+	if _, err := m.ds.GetOrphanRO(name); err == nil || !apierrors.IsNotFound(err) {
+		return nil, err
+	}
+
+	// labels will be attached by mutator webhook
+	orphan := &longhorn.Orphan{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:            name,
+			OwnerReferences: datastore.GetOwnerReferencesForInstanceManager(im),
+		},
+		Spec: longhorn.OrphanSpec{
+			NodeID:     m.controllerID,
+			Type:       orphanType,
+			DataEngine: dataEngineType,
+			Parameters: map[string]string{
+				longhorn.OrphanInstanceName:    instanceName,
+				longhorn.OrphanInstanceManager: im.Name,
+			},
+		},
+	}
+
+	return m.ds.CreateOrphan(orphan)
+}
+
+func (m *InstanceManagerMonitor) categorizeProcesses(instanceMap instanceProcessMap) (instanceProcessMap, instanceProcessMap) {
+	engineProcesses := make(instanceProcessMap)
+	replicaProcesses := make(instanceProcessMap)
+	for name, process := range instanceMap {
+		switch process.Status.Type {
+		case longhorn.InstanceTypeEngine:
+			engineProcesses[name] = process
+		case longhorn.InstanceTypeReplica:
+			replicaProcesses[name] = process
+		}
+	}
+	return engineProcesses, replicaProcesses
+}
+
 func (imc *InstanceManagerController) isResponsibleFor(im *longhorn.InstanceManager) bool {
 	return isControllerResponsibleFor(imc.controllerID, imc.ds, im.Name, im.Spec.NodeID, im.Status.OwnerID)
 }
diff --git a/controller/node_controller.go b/controller/node_controller.go
index 592796f6ba..9f23a89522 100644
--- a/controller/node_controller.go
+++ b/controller/node_controller.go
@@ -194,7 +194,7 @@ func (nc *NodeController) isResponsibleForSetting(obj interface{}) bool {
 
 	return types.SettingName(setting.Name) == types.SettingNameStorageMinimalAvailablePercentage ||
 		types.SettingName(setting.Name) == types.SettingNameBackingImageCleanupWaitInterval ||
-		types.SettingName(setting.Name) == types.SettingNameOrphanAutoDeletion ||
+		types.SettingName(setting.Name) == types.SettingNameOrphanResourceAutoDeletion ||
 		types.SettingName(setting.Name) == types.SettingNameNodeDrainPolicy
 }
 
@@ -1328,7 +1328,9 @@ func (nc *NodeController) getNewAndMissingOrphanedReplicaDataStores(diskName, di
 	}
 	orphanMap := make(map[string]*longhorn.Orphan, len(orphanList))
 	for _, o := range orphanList {
-		orphanMap[o.Name] = o
+		if o.Spec.Type == longhorn.OrphanTypeReplicaData {
+			orphanMap[o.Name] = o
+		}
 	}
 
 	for dataStore := range replicaDataStores {
@@ -1355,10 +1357,11 @@ func (nc *NodeController) getNewAndMissingOrphanedReplicaDataStores(diskName, di
 }
 
 func (nc *NodeController) deleteOrphans(node *longhorn.Node, diskName string, diskInfo *monitor.CollectedDiskInfo, missingOrphanedReplicaDataStores map[string]string) error {
-	autoDeletionEnabled, err := nc.ds.GetSettingAsBool(types.SettingNameOrphanAutoDeletion)
+	autoDeletionResourceTypes, err := nc.ds.GetSettingOrphanResourceAutoDeletion()
 	if err != nil {
-		return errors.Wrapf(err, "failed to get %v setting", types.SettingNameOrphanAutoDeletion)
+		return errors.Wrapf(err, "failed to get %v setting", types.SettingNameOrphanResourceAutoDeletion)
 	}
+	autoDeletionEnabled := autoDeletionResourceTypes[types.OrphanResourceTypeReplicaData]
 
 	for dataStore := range missingOrphanedReplicaDataStores {
 		orphanName := types.GetOrphanChecksumNameForOrphanedDataStore(node.Name, diskName, diskInfo.Path, diskInfo.DiskUUID, dataStore)
@@ -1415,7 +1418,7 @@ func (nc *NodeController) createOrphan(node *longhorn.Node, diskName, replicaDat
 		},
 		Spec: longhorn.OrphanSpec{
 			NodeID: node.Name,
-			Type:   longhorn.OrphanTypeReplica,
+			Type:   longhorn.OrphanTypeReplicaData,
 			Parameters: map[string]string{
 				longhorn.OrphanDataName: replicaDataStore,
 				longhorn.OrphanDiskName: diskName,
diff --git a/controller/node_controller_test.go b/controller/node_controller_test.go
index 6f11faecb3..19dc29b30c 100644
--- a/controller/node_controller_test.go
+++ b/controller/node_controller_test.go
@@ -40,7 +40,7 @@ var (
 	DefaultOrphanTestNode1 = newOrphan(
 		longhorn.OrphanSpec{
 			NodeID: TestNode1,
-			Type:   longhorn.OrphanTypeReplica,
+			Type:   longhorn.OrphanTypeReplicaData,
 			Parameters: map[string]string{
 				longhorn.OrphanDataName: monitor.TestOrphanedReplicaDirectoryName,
 				longhorn.OrphanDiskName: TestDiskID1,
diff --git a/controller/orphan_controller.go b/controller/orphan_controller.go
index 8538eb0970..9a4202bf44 100644
--- a/controller/orphan_controller.go
+++ b/controller/orphan_controller.go
@@ -17,7 +17,6 @@ import (
 	"k8s.io/kubernetes/pkg/controller"
 
 	corev1 "k8s.io/api/core/v1"
-	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	clientset "k8s.io/client-go/kubernetes"
 	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
@@ -93,6 +92,15 @@ func NewOrphanController(
 	}
 	oc.cacheSyncs = append(oc.cacheSyncs, ds.NodeInformer.HasSynced)
 
+	if _, err = ds.InstanceManagerInformer.AddEventHandlerWithResyncPeriod(cache.ResourceEventHandlerFuncs{
+		AddFunc:    func(cur interface{}) { oc.enqueueForInstanceManager(cur) },
+		UpdateFunc: func(old, cur interface{}) { oc.enqueueForInstanceManager(cur) },
+		DeleteFunc: func(cur interface{}) { oc.enqueueForInstanceManager(cur) },
+	}, 0); err != nil {
+		return nil, err
+	}
+	oc.cacheSyncs = append(oc.cacheSyncs, ds.InstanceManagerInformer.HasSynced)
+
 	return oc, nil
 }
 
@@ -133,6 +141,33 @@ func (oc *OrphanController) enqueueForLonghornNode(obj interface{}) {
 	}
 }
 
+func (oc *OrphanController) enqueueForInstanceManager(obj interface{}) {
+	im, ok := obj.(*longhorn.InstanceManager)
+	if !ok {
+		deletedState, ok := obj.(cache.DeletedFinalStateUnknown)
+		if !ok {
+			utilruntime.HandleError(fmt.Errorf("received unexpected obj: %#v", obj))
+			return
+		}
+		// use the last known state, to enqueue, dependent objects
+		im, ok = deletedState.Obj.(*longhorn.InstanceManager)
+		if !ok {
+			utilruntime.HandleError(fmt.Errorf("DeletedFinalStateUnknown contained invalid object: %#v", deletedState.Obj))
+			return
+		}
+	}
+
+	orphans, err := oc.ds.ListOrphansByNodeRO(im.Spec.NodeID)
+	if err != nil {
+		utilruntime.HandleError(fmt.Errorf("failed to list orphans on instance manager %v since %v", im.Name, err))
+		return
+	}
+
+	for _, orphan := range orphans {
+		oc.enqueueOrphan(orphan)
+	}
+}
+
 func (oc *OrphanController) Run(workers int, stopCh <-chan struct{}) {
 	defer utilruntime.HandleCrash()
 	defer oc.queue.ShutDown()
@@ -201,7 +236,7 @@ func (oc *OrphanController) syncOrphan(key string) (err error) {
 func (oc *OrphanController) reconcile(orphanName string) (err error) {
 	orphan, err := oc.ds.GetOrphan(orphanName)
 	if err != nil {
-		if !apierrors.IsNotFound(err) {
+		if !datastore.ErrorIsNotFound(err) {
 			return err
 		}
 		return nil
@@ -218,7 +253,7 @@ func (oc *OrphanController) reconcile(orphanName string) (err error) {
 		orphan, err = oc.ds.UpdateOrphanStatus(orphan)
 		if err != nil {
 			// we don't mind others coming first
-			if apierrors.IsConflict(errors.Cause(err)) {
+			if datastore.ErrorIsConflict(errors.Cause(err)) {
 				return nil
 			}
 			return err
@@ -227,13 +262,11 @@ func (oc *OrphanController) reconcile(orphanName string) (err error) {
 	}
 
 	if !orphan.DeletionTimestamp.IsZero() {
-		defer func() {
-			if err == nil {
-				err = oc.ds.RemoveFinalizerForOrphan(orphan)
-			}
-		}()
-
-		return oc.cleanupOrphanedData(orphan)
+		isCleanupComplete, err := oc.cleanupOrphanedResource(orphan)
+		if isCleanupComplete {
+			return oc.ds.RemoveFinalizerForOrphan(orphan)
+		}
+		return err
 	}
 
 	existingOrphan := orphan.DeepCopy()
@@ -244,7 +277,7 @@ func (oc *OrphanController) reconcile(orphanName string) (err error) {
 		if reflect.DeepEqual(existingOrphan.Status, orphan.Status) {
 			return
 		}
-		if _, err := oc.ds.UpdateOrphanStatus(orphan); err != nil && apierrors.IsConflict(errors.Cause(err)) {
+		if _, err := oc.ds.UpdateOrphanStatus(orphan); err != nil && datastore.ErrorIsConflict(errors.Cause(err)) {
 			log.WithError(err).Debugf("Requeue %v due to conflict", orphanName)
 			oc.enqueueOrphan(orphan)
 		}
@@ -269,7 +302,7 @@ func (oc *OrphanController) isResponsibleFor(orphan *longhorn.Orphan) bool {
 	return isControllerResponsibleFor(oc.controllerID, oc.ds, orphan.Name, orphan.Spec.NodeID, orphan.Status.OwnerID)
 }
 
-func (oc *OrphanController) cleanupOrphanedData(orphan *longhorn.Orphan) (err error) {
+func (oc *OrphanController) cleanupOrphanedResource(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
 	log := getLoggerForOrphan(oc.logger, orphan)
 
 	defer func() {
@@ -277,7 +310,7 @@ func (oc *OrphanController) cleanupOrphanedData(orphan *longhorn.Orphan) (err er
 			return
 		}
 
-		err = errors.Wrapf(err, "failed to delete orphan %v data", orphan.Name)
+		err = errors.Wrapf(err, "failed to delete %v orphan %v", orphan.Spec.Type, orphan.Name)
 		orphan.Status.Conditions = types.SetCondition(orphan.Status.Conditions,
 			longhorn.OrphanConditionTypeError, longhorn.ConditionStatusTrue, "", err.Error())
 	}()
@@ -285,32 +318,191 @@ func (oc *OrphanController) cleanupOrphanedData(orphan *longhorn.Orphan) (err er
 	// Make sure if the orphan nodeID and controller ID are same.
 	// If NO, just delete the orphan resource object and don't touch the data.
 	if orphan.Spec.NodeID != oc.controllerID {
-		log.Infof("Orphan nodeID %v is different from controllerID %v, so just delete the orphan resource object",
-			orphan.Name, oc.controllerID)
-		return nil
+		log.WithFields(logrus.Fields{
+			"orphanType": orphan.Spec.Type,
+			"orphanName": orphan.Name,
+			"orphanNode": orphan.Spec.NodeID,
+		}).Infof("Orphan does not belong to this controller. Skipping resource cleanup")
+		return true, nil
 	}
 
-	if types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeDataCleanable).Status !=
-		longhorn.ConditionStatusTrue {
-		log.Infof("Only delete orphan %v resource object and do not delete the orphaned data", orphan.Name)
-		return nil
+	switch orphan.Spec.Type {
+	case longhorn.OrphanTypeEngineInstance:
+		isCleanupComplete, err = oc.cleanupOrphanedEngineInstance(orphan)
+	case longhorn.OrphanTypeReplicaInstance:
+		isCleanupComplete, err = oc.cleanupOrphanedReplicaInstance(orphan)
+	case longhorn.OrphanTypeReplicaData:
+		if types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeDataCleanable).Status !=
+			longhorn.ConditionStatusTrue {
+			log.Infof("Only delete orphan %v resource object and do not delete the orphaned data store", orphan.Name)
+			return true, nil
+		}
+		err = oc.deleteOrphanedReplicaDataStore(orphan)
+		if err == nil || (err != nil && datastore.ErrorIsNotFound(err)) {
+			isCleanupComplete = true
+			err = nil
+		}
+	default:
+		return false, fmt.Errorf("unknown orphan type %v to clean up orphaned resource for %v", orphan.Spec.Type, orphan.Name)
 	}
 
-	switch orphan.Spec.Type {
-	case longhorn.OrphanTypeReplica:
-		err = oc.deleteOrphanedReplica(orphan)
+	return isCleanupComplete, err
+}
+
+func (oc *OrphanController) cleanupOrphanedEngineInstance(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
+	defer func() {
+		if err != nil {
+			err = errors.Wrapf(err, "failed to cleanup orphan engine instance %v", orphan.Name)
+		}
+	}()
+
+	instance, imName, err := oc.extractOrphanedInstanceInfo(orphan)
+	if err != nil {
+		return false, err
+	}
+
+	var spec *longhorn.InstanceSpec
+	if engineCR, err := oc.ds.GetEngineRO(instance); err != nil {
+		if !datastore.ErrorIsNotFound(err) {
+			return false, err
+		}
+		spec = nil
+	} else {
+		spec = &engineCR.Spec.InstanceSpec
+	}
+	return oc.cleanupOrphanedInstance(orphan, instance, imName, longhorn.InstanceManagerTypeEngine, spec)
+}
+
+func (oc *OrphanController) cleanupOrphanedReplicaInstance(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
+	defer func() {
+		if err != nil {
+			err = errors.Wrapf(err, "failed to cleanup orphan replica instance %v", orphan.Name)
+		}
+	}()
+
+	instance, imName, err := oc.extractOrphanedInstanceInfo(orphan)
+	if err != nil {
+		return false, err
+	}
+
+	var spec *longhorn.InstanceSpec
+	if replicaCR, err := oc.ds.GetReplicaRO(instance); err != nil {
+		if !datastore.ErrorIsNotFound(err) {
+			return false, err
+		}
+		spec = nil
+	} else {
+		spec = &replicaCR.Spec.InstanceSpec
+	}
+	return oc.cleanupOrphanedInstance(orphan, instance, imName, longhorn.InstanceManagerTypeReplica, spec)
+}
+
+func (oc *OrphanController) extractOrphanedInstanceInfo(orphan *longhorn.Orphan) (name, instanceManager string, err error) {
+	name, ok := orphan.Spec.Parameters[longhorn.OrphanInstanceName]
+	if !ok {
+		return "", "", fmt.Errorf("failed to get instance name for instance orphan %v", orphan.Name)
+	}
+
+	instanceManager, ok = orphan.Spec.Parameters[longhorn.OrphanInstanceManager]
+	if !ok {
+		return "", "", fmt.Errorf("failed to get instance manager for instance orphan %v", orphan.Name)
+	}
+
+	switch orphan.Spec.DataEngine {
+	case longhorn.DataEngineTypeV1, longhorn.DataEngineTypeV2:
+		// supported data engine type
 	default:
-		err = fmt.Errorf("unknown orphan type %v", orphan.Spec.Type)
+		return "", "", fmt.Errorf("unknown data engine type %v for instance orphan %v", orphan.Spec.DataEngine, orphan.Name)
 	}
 
-	if err == nil || (err != nil && apierrors.IsNotFound(err)) {
-		return nil
+	return name, instanceManager, nil
+}
+
+func (oc *OrphanController) cleanupOrphanedInstance(orphan *longhorn.Orphan, instance, imName string, imType longhorn.InstanceManagerType, instanceCRSpec *longhorn.InstanceSpec) (isCleanupComplete bool, err error) {
+	if instanceCRSpec != nil && instanceCRSpec.NodeID == orphan.Spec.NodeID {
+		oc.logger.Infof("Orphan instance %v is scheduled back to current node %v. Skip cleaning up the instance resource and finalize the orphan CR.", instance, orphan.Spec.NodeID)
+		return true, nil
 	}
 
-	return err
+	imc, err := oc.getRunningInstanceManagerClientForOrphan(orphan, imName)
+	if err != nil {
+		return false, errors.Wrapf(err, "failed to get running instance manager client for orphan %v", orphan.Name)
+	} else if imc == nil {
+		oc.logger.WithField("orphanInstanceNode", orphan.Spec.NodeID).Warnf("No running instance manager for deleting orphan instance %v", orphan.Name)
+		return true, nil
+	}
+	defer func() {
+		if closeErr := imc.Close(); closeErr != nil {
+			oc.logger.WithError(closeErr).Error("Failed to close instance manager client")
+		}
+	}()
+
+	if err := oc.deleteInstance(imc, instance, imType, orphan.Spec.DataEngine); err != nil {
+		return false, err
+	}
+	isCleanupComplete, cleanupErr := oc.confirmOrphanInstanceCleanup(imc, instance, imType, orphan.Spec.DataEngine)
+	if cleanupErr == nil && !isCleanupComplete {
+		oc.logger.Infof("Orphan instance %v cleanup in progress", instance)
+	}
+	return isCleanupComplete, cleanupErr
+}
+
+func (oc *OrphanController) getRunningInstanceManagerClientForOrphan(orphan *longhorn.Orphan, imName string) (*engineapi.InstanceManagerClient, error) {
+	im, err := oc.ds.GetRunningInstanceManagerByNodeRO(orphan.Spec.NodeID, orphan.Spec.DataEngine)
+	if err != nil {
+		if datastore.ErrorIsNotFound(err) {
+			return nil, nil
+		}
+		return nil, err
+	}
+	if im.Name != imName {
+		oc.logger.WithField("instanceManager", imName).Warnf("Orphan instance %v is not managed by current running instance manager %v", orphan.Name, im.Name)
+		return nil, nil
+	}
+	return engineapi.NewInstanceManagerClient(im, false)
+}
+
+func (oc *OrphanController) deleteInstance(imc *engineapi.InstanceManagerClient, instanceName string, instanceKind longhorn.InstanceManagerType, engineType longhorn.DataEngineType) (err error) {
+	// There is a delay between deletion initiation and state/InstanceManager update,
+	// this function may be called multiple times before given instance exits.
+
+	oc.logger.Infof("Orphan controller deleting instance %v", instanceName)
+
+	defer func() {
+		err = errors.Wrapf(err, "failed to delete instance %v", instanceName)
+	}()
+
+	err = imc.InstanceDelete(engineType, instanceName, string(instanceKind), "", false)
+	if err != nil && !types.ErrorIsNotFound(err) {
+		return err
+	}
+
+	return nil
+}
+
+func (oc *OrphanController) confirmOrphanInstanceCleanup(imc *engineapi.InstanceManagerClient, instanceName string, imType longhorn.InstanceManagerType, engineType longhorn.DataEngineType) (isCleanupComplete bool, err error) {
+	defer func() {
+		if err != nil {
+			err = errors.Wrapf(err, "failed to confirm cleanup result of instance %v", instanceName)
+		}
+	}()
+
+	_, err = imc.InstanceGet(engineType, instanceName, string(imType))
+	switch {
+	case err == nil:
+		// Instance still exists - cleanup not complete
+		// Cleanup will continue after the instance state update.
+		return false, nil
+	case types.ErrorIsNotFound(err):
+		// instance not found - cleanup completed.
+		return true, nil
+	default:
+		// Unexpected error - cleanup status unknown.
+		return false, err
+	}
 }
 
-func (oc *OrphanController) deleteOrphanedReplica(orphan *longhorn.Orphan) error {
+func (oc *OrphanController) deleteOrphanedReplicaDataStore(orphan *longhorn.Orphan) error {
 	oc.logger.Infof("Deleting orphan %v replica data store %v in disk %v on node %v",
 		orphan.Name, orphan.Spec.Parameters[longhorn.OrphanDataName],
 		orphan.Spec.Parameters[longhorn.OrphanDiskPath], orphan.Status.OwnerID)
@@ -360,8 +552,21 @@ func (oc *OrphanController) DeleteV2ReplicaInstance(diskName, diskUUID, diskDriv
 }
 
 func (oc *OrphanController) updateConditions(orphan *longhorn.Orphan) error {
-	if err := oc.updateDataCleanableCondition(orphan); err != nil {
-		return err
+	switch orphan.Spec.Type {
+	case longhorn.OrphanTypeEngineInstance:
+		if err := oc.updateInstanceStateCondition(orphan, longhorn.InstanceTypeEngine); err != nil {
+			return err
+		}
+	case longhorn.OrphanTypeReplicaInstance:
+		if err := oc.updateInstanceStateCondition(orphan, longhorn.InstanceTypeReplica); err != nil {
+			return err
+		}
+	case longhorn.OrphanTypeReplicaData:
+		if err := oc.updateDataCleanableCondition(orphan); err != nil {
+			return err
+		}
+	default:
+		return fmt.Errorf("unknown orphan type %v to update conditions on orphan %v", orphan.Spec.Type, orphan.Name)
 	}
 
 	if types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeError).Status != longhorn.ConditionStatusTrue {
@@ -371,6 +576,55 @@ func (oc *OrphanController) updateConditions(orphan *longhorn.Orphan) error {
 	return nil
 }
 
+// If the InstanceState condition status is true, the corresponding instance is still exist in instance manager
+func (oc *OrphanController) updateInstanceStateCondition(orphan *longhorn.Orphan, instanceType longhorn.InstanceType) (err error) {
+	var instanceState longhorn.InstanceState
+	defer func() {
+		if err != nil {
+			return
+		}
+
+		status := longhorn.ConditionStatusTrue
+		if instanceState == longhorn.InstanceStateTerminated {
+			status = longhorn.ConditionStatusFalse
+		}
+
+		orphan.Status.Conditions = types.SetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeInstanceExist, status, string(instanceState), "")
+	}()
+
+	instanceName, instanceManager, err := oc.extractOrphanedInstanceInfo(orphan)
+	im, err := oc.ds.GetInstanceManager(instanceManager)
+	if err != nil {
+		if datastore.ErrorIsNotFound(err) {
+			oc.logger.WithError(err).Infof("No instance manager for node %v for update instance state of orphan instance %v", oc.controllerID, orphan.Name)
+			instanceState = longhorn.InstanceStateTerminated
+			return nil
+		}
+		return err
+	}
+	imc, err := engineapi.NewInstanceManagerClient(im, false)
+	if err != nil {
+		return errors.Wrapf(err, "failed to check node down or missing manager for node %v", orphan.Spec.NodeID)
+	}
+	defer func() {
+		if closeErr := imc.Close(); closeErr != nil {
+			oc.logger.WithError(closeErr).Error("Failed to close instance manager client")
+		}
+	}()
+
+	instance, err := imc.InstanceGet(orphan.Spec.DataEngine, instanceName, string(instanceType))
+	switch {
+	case err != nil:
+		return errors.Wrapf(err, "failed to get instance %v", instanceName)
+	case instance == nil:
+		instanceState = longhorn.InstanceStateTerminated
+	default:
+		instanceState = instance.Status.State
+	}
+
+	return nil
+}
+
 // If the DataCleanable condition status is false, then the deletion of the orphan CRD won't delete the on-disk data
 func (oc *OrphanController) updateDataCleanableCondition(orphan *longhorn.Orphan) (err error) {
 	reason := ""
@@ -399,7 +653,7 @@ func (oc *OrphanController) updateDataCleanableCondition(orphan *longhorn.Orphan
 
 	node, err := oc.ds.GetNode(orphan.Spec.NodeID)
 	if err != nil {
-		if !apierrors.IsNotFound(err) {
+		if !datastore.ErrorIsNotFound(err) {
 			return fmt.Errorf("failed to get node %v", orphan.Spec.NodeID)
 		}
 		reason = longhorn.OrphanConditionTypeDataCleanableReasonNodeUnavailable
@@ -411,7 +665,7 @@ func (oc *OrphanController) updateDataCleanableCondition(orphan *longhorn.Orphan
 		return nil
 	}
 
-	if orphan.Spec.Type == longhorn.OrphanTypeReplica {
+	if orphan.Spec.Type == longhorn.OrphanTypeReplicaData {
 		reason = oc.checkOrphanedReplicaDataCleanable(node, orphan)
 	}
 
diff --git a/controller/setting_controller.go b/controller/setting_controller.go
index 46db06ff02..f519e843ad 100644
--- a/controller/setting_controller.go
+++ b/controller/setting_controller.go
@@ -256,6 +256,10 @@ func (sc *SettingController) syncNonDangerZoneSettingsForManagedComponents(setti
 		if err := sc.syncDefaultLonghornStaticStorageClass(); err != nil {
 			return err
 		}
+	case types.SettingNameOrphanResourceAutoDeletion:
+		if err := sc.syncOrphanResourceAutoDeletion(); err != nil {
+			return err
+		}
 	}
 
 	return nil
@@ -896,6 +900,32 @@ func (sc *SettingController) syncDefaultLonghornStaticStorageClass() error {
 	return err
 }
 
+func (sc *SettingController) syncOrphanResourceAutoDeletion() error {
+	setting, err := sc.ds.GetSettingOrphanResourceAutoDeletion()
+	if err != nil {
+		return err
+	}
+	var replicaDataCleanupValue string
+	if setting[types.OrphanResourceTypeReplicaData] {
+		replicaDataCleanupValue = "true"
+	} else {
+		replicaDataCleanupValue = "false"
+	}
+
+	deprecatedSetting, err := sc.ds.GetSetting(types.SettingNameOrphanAutoDeletion)
+	if err != nil {
+		return err
+	}
+	if replicaDataCleanupValue != deprecatedSetting.Value {
+		deprecatedSetting.Value = replicaDataCleanupValue
+		if _, err := sc.ds.UpdateSetting(deprecatedSetting); err != nil {
+			return err
+		}
+	}
+
+	return err
+}
+
 // updateDataEngine deletes the corresponding instance manager pods immediately if the data engine setting is disabled.
 func (sc *SettingController) updateDataEngine(setting types.SettingName) error {
 	enabled, err := sc.ds.GetSettingAsBool(setting)
@@ -1585,6 +1615,7 @@ func (info *ClusterInfo) collectSettings() error {
 		types.SettingNameNodeDownPodDeletionPolicy:                                true,
 		types.SettingNameNodeDrainPolicy:                                          true,
 		types.SettingNameOrphanAutoDeletion:                                       true,
+		types.SettingNameOrphanResourceAutoDeletion:                               true,
 		types.SettingNameRecurringFailedJobsHistoryLimit:                          true,
 		types.SettingNameRecurringSuccessfulJobsHistoryLimit:                      true,
 		types.SettingNameRemoveSnapshotsDuringFilesystemTrim:                      true,
diff --git a/datastore/datastore.go b/datastore/datastore.go
index e933165592..ca2ea7c693 100644
--- a/datastore/datastore.go
+++ b/datastore/datastore.go
@@ -1,6 +1,7 @@
 package datastore
 
 import (
+	"errors"
 	"time"
 
 	"k8s.io/client-go/tools/cache"
@@ -16,6 +17,7 @@ import (
 	schedulinglisters "k8s.io/client-go/listers/scheduling/v1"
 	storagelisters_v1 "k8s.io/client-go/listers/storage/v1"
 
+	"github.com/longhorn/longhorn-manager/types"
 	"github.com/longhorn/longhorn-manager/util"
 
 	lhclientset "github.com/longhorn/longhorn-manager/k8s/pkg/client/clientset/versioned"
@@ -318,7 +320,7 @@ func (s *DataStore) Sync(stopCh <-chan struct{}) bool {
 // ErrorIsNotFound checks if given error match
 // metav1.StatusReasonNotFound
 func ErrorIsNotFound(err error) bool {
-	return apierrors.IsNotFound(err)
+	return errors.Is(err, &types.NotFoundError{}) || apierrors.IsNotFound(err)
 }
 
 // ErrorIsConflict checks if given error match
diff --git a/datastore/longhorn.go b/datastore/longhorn.go
index 7a3997cb16..50a6126124 100644
--- a/datastore/longhorn.go
+++ b/datastore/longhorn.go
@@ -3523,6 +3523,21 @@ func (s *DataStore) GetSettingSystemManagedComponentsNodeSelector() (map[string]
 	return nodeSelector, nil
 }
 
+// GetSettingOrphanResourceAutoDeletion get the setting and return a flag collection of orphaned resource types.
+// Flag is true when the auto deletion is enabled to an orphaned resource type.
+// Returns error if the setting is invalid
+func (s *DataStore) GetSettingOrphanResourceAutoDeletion() (map[types.OrphanResourceType]bool, error) {
+	setting, err := s.GetSettingWithAutoFillingRO(types.SettingNameOrphanResourceAutoDeletion)
+	if err != nil {
+		return nil, err
+	}
+	resourceTypes, err := types.UnmarshalOrphanResourceTypes(setting.Value)
+	if err != nil {
+		return nil, err
+	}
+	return resourceTypes, nil
+}
+
 // ResetMonitoringEngineStatus clean and update Engine status
 func (s *DataStore) ResetMonitoringEngineStatus(e *longhorn.Engine) (*longhorn.Engine, error) {
 	e.Status.Endpoint = ""
@@ -5255,6 +5270,23 @@ func (s *DataStore) ListOrphansByNodeRO(name string) ([]*longhorn.Orphan, error)
 	return s.orphanLister.Orphans(s.namespace).List(nodeSelector)
 }
 
+// ListOrphansForEngineAndReplicaInstancesRO returns a list of all engine and replica instance Orphans on node Name for the given namespace,
+// the list contains direct references to the internal cache objects and should not be mutated.
+// Consider using this function when you can guarantee read only access and don't want the overhead of deep copies
+func (s *DataStore) ListOrphansForEngineAndReplicaInstancesRO(name string) (orphanList []*longhorn.Orphan, err error) {
+	existOrphans, err := s.ListOrphansByNode(name)
+	if err != nil {
+		return nil, err
+	}
+	for _, orphan := range existOrphans {
+		switch orphan.Spec.Type {
+		case longhorn.OrphanTypeEngineInstance, longhorn.OrphanTypeReplicaInstance:
+			orphanList = append(orphanList, orphan)
+		}
+	}
+	return orphanList, nil
+}
+
 // DeleteOrphan won't result in immediately deletion since finalizer was set by default
 func (s *DataStore) DeleteOrphan(orphanName string) error {
 	return s.lhClient.LonghornV1beta2().Orphans(s.namespace).Delete(context.TODO(), orphanName, metav1.DeleteOptions{})
@@ -6072,7 +6104,7 @@ func (s *DataStore) GetRunningInstanceManagerByNodeRO(node string, dataEngine lo
 		}
 	}
 
-	return nil, fmt.Errorf("failed to find a running instance manager for node %v", node)
+	return nil, &types.NotFoundError{Name: "a running instance manager for node " + node}
 }
 
 func (s *DataStore) GetFreezeFilesystemForSnapshotSetting(e *longhorn.Engine) (bool, error) {
diff --git a/k8s/crds.yaml b/k8s/crds.yaml
index fc77a8f373..5350055833 100644
--- a/k8s/crds.yaml
+++ b/k8s/crds.yaml
@@ -2709,6 +2709,14 @@ spec:
             description: OrphanSpec defines the desired state of the Longhorn orphaned
               data
             properties:
+              dataEngine:
+                description: |-
+                  The type of data engine for instance orphan.
+                  Can be "v1", "v2".
+                enum:
+                - v1
+                - v2
+                type: string
               nodeID:
                 description: The node ID on which the controller is responsible to
                   reconcile this orphan CR.
diff --git a/k8s/pkg/apis/longhorn/v1beta2/instancemanager.go b/k8s/pkg/apis/longhorn/v1beta2/instancemanager.go
index d903ee4228..d412b90e4e 100644
--- a/k8s/pkg/apis/longhorn/v1beta2/instancemanager.go
+++ b/k8s/pkg/apis/longhorn/v1beta2/instancemanager.go
@@ -61,6 +61,8 @@ const (
 	InstanceStateStarting = InstanceState("starting")
 	InstanceStateStopping = InstanceState("stopping")
 	InstanceStateUnknown  = InstanceState("unknown")
+	// InstanceStateTerminated indicates the instance is deleted and not found
+	InstanceStateTerminated = InstanceState("terminated")
 )
 
 type InstanceSpec struct {
diff --git a/k8s/pkg/apis/longhorn/v1beta2/orphan.go b/k8s/pkg/apis/longhorn/v1beta2/orphan.go
index 9eede7bfdb..34273bd085 100644
--- a/k8s/pkg/apis/longhorn/v1beta2/orphan.go
+++ b/k8s/pkg/apis/longhorn/v1beta2/orphan.go
@@ -5,12 +5,15 @@ import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 type OrphanType string
 
 const (
-	OrphanTypeReplica = OrphanType("replica")
+	OrphanTypeReplicaData     = OrphanType("replica")
+	OrphanTypeEngineInstance  = OrphanType("engine-instance")
+	OrphanTypeReplicaInstance = OrphanType("replica-instance")
 )
 
 const (
 	OrphanConditionTypeError         = "Error"
 	OrphanConditionTypeDataCleanable = "DataCleanable"
+	OrphanConditionTypeInstanceExist = "InstanceExist"
 
 	OrphanConditionTypeDataCleanableReasonNodeUnavailable = "NodeUnavailable"
 	OrphanConditionTypeDataCleanableReasonNodeEvicted     = "NodeEvicted"
@@ -20,6 +23,9 @@ const (
 )
 
 const (
+	OrphanInstanceName    = "InstanceName"
+	OrphanInstanceManager = "InstanceManager"
+
 	OrphanDataName = "DataName"
 	OrphanDiskName = "DiskName"
 	OrphanDiskUUID = "DiskUUID"
@@ -36,6 +42,11 @@ type OrphanSpec struct {
 	// Can be "replica".
 	// +optional
 	Type OrphanType `json:"orphanType"`
+	// The type of data engine for instance orphan.
+	// Can be "v1", "v2".
+	// +optional
+	// +kubebuilder:validation:Enum=v1;v2
+	DataEngine DataEngineType `json:"dataEngine,omitempty"`
 
 	// The parameters of the orphaned data
 	// +optional
diff --git a/k8s/pkg/client/applyconfiguration/longhorn/v1beta2/orphanspec.go b/k8s/pkg/client/applyconfiguration/longhorn/v1beta2/orphanspec.go
index 10fda6d6f8..15bc343726 100644
--- a/k8s/pkg/client/applyconfiguration/longhorn/v1beta2/orphanspec.go
+++ b/k8s/pkg/client/applyconfiguration/longhorn/v1beta2/orphanspec.go
@@ -25,9 +25,10 @@ import (
 // OrphanSpecApplyConfiguration represents a declarative configuration of the OrphanSpec type for use
 // with apply.
 type OrphanSpecApplyConfiguration struct {
-	NodeID     *string                     `json:"nodeID,omitempty"`
-	Type       *longhornv1beta2.OrphanType `json:"orphanType,omitempty"`
-	Parameters map[string]string           `json:"parameters,omitempty"`
+	NodeID     *string                         `json:"nodeID,omitempty"`
+	Type       *longhornv1beta2.OrphanType     `json:"orphanType,omitempty"`
+	DataEngine *longhornv1beta2.DataEngineType `json:"dataEngine,omitempty"`
+	Parameters map[string]string               `json:"parameters,omitempty"`
 }
 
 // OrphanSpecApplyConfiguration constructs a declarative configuration of the OrphanSpec type for use with
@@ -52,6 +53,14 @@ func (b *OrphanSpecApplyConfiguration) WithType(value longhornv1beta2.OrphanType
 	return b
 }
 
+// WithDataEngine sets the DataEngine field in the declarative configuration to the given value
+// and returns the receiver, so that objects can be built by chaining "With" function invocations.
+// If called multiple times, the DataEngine field is set to the value of the last call.
+func (b *OrphanSpecApplyConfiguration) WithDataEngine(value longhornv1beta2.DataEngineType) *OrphanSpecApplyConfiguration {
+	b.DataEngine = &value
+	return b
+}
+
 // WithParameters puts the entries into the Parameters field in the declarative configuration
 // and returns the receiver, so that objects can be build by chaining "With" function invocations.
 // If called multiple times, the entries provided by each call will be put on the Parameters field,
diff --git a/types/setting.go b/types/setting.go
index 3ec147afa1..553ad1dcc5 100644
--- a/types/setting.go
+++ b/types/setting.go
@@ -100,6 +100,7 @@ const (
 	SettingNameGuaranteedInstanceManagerCPU                             = SettingName("guaranteed-instance-manager-cpu")
 	SettingNameKubernetesClusterAutoscalerEnabled                       = SettingName("kubernetes-cluster-autoscaler-enabled")
 	SettingNameOrphanAutoDeletion                                       = SettingName("orphan-auto-deletion")
+	SettingNameOrphanResourceAutoDeletion                               = SettingName("orphan-resource-auto-deletion")
 	SettingNameStorageNetwork                                           = SettingName("storage-network")
 	SettingNameStorageNetworkForRWXVolumeEnabled                        = SettingName("storage-network-for-rwx-volume-enabled")
 	SettingNameFailedBackupTTL                                          = SettingName("failed-backup-ttl")
@@ -202,6 +203,7 @@ var (
 		SettingNameGuaranteedInstanceManagerCPU,
 		SettingNameKubernetesClusterAutoscalerEnabled,
 		SettingNameOrphanAutoDeletion,
+		SettingNameOrphanResourceAutoDeletion,
 		SettingNameStorageNetwork,
 		SettingNameStorageNetworkForRWXVolumeEnabled,
 		SettingNameFailedBackupTTL,
@@ -326,6 +328,7 @@ var (
 		SettingNameGuaranteedInstanceManagerCPU:                             SettingDefinitionGuaranteedInstanceManagerCPU,
 		SettingNameKubernetesClusterAutoscalerEnabled:                       SettingDefinitionKubernetesClusterAutoscalerEnabled,
 		SettingNameOrphanAutoDeletion:                                       SettingDefinitionOrphanAutoDeletion,
+		SettingNameOrphanResourceAutoDeletion:                               SettingDefinitionOrphanResourceAutoDeletion,
 		SettingNameStorageNetwork:                                           SettingDefinitionStorageNetwork,
 		SettingNameStorageNetworkForRWXVolumeEnabled:                        SettingDefinitionStorageNetworkForRWXVolumeEnabled,
 		SettingNameFailedBackupTTL:                                          SettingDefinitionFailedBackupTTL,
@@ -1055,14 +1058,31 @@ var (
 	SettingDefinitionOrphanAutoDeletion = SettingDefinition{
 		DisplayName: "Orphan Auto-Deletion",
 		Description: "This setting allows Longhorn to delete the orphan resource and its corresponding orphaned data automatically. \n\n" +
-			"Orphan resources on down or unknown nodes will not be cleaned up automatically. \n\n",
+			"Orphan resources on down or unknown nodes will not be cleaned up automatically. \n\n" +
+			fmt.Sprintf("Deprecated: enable \"%s\" in %s instead. \n\n", OrphanResourceTypeReplicaData, SettingNameOrphanResourceAutoDeletion),
 		Category: SettingCategoryOrphan,
 		Type:     SettingTypeBool,
 		Required: true,
-		ReadOnly: false,
+		ReadOnly: true,
 		Default:  "false",
 	}
 
+	SettingDefinitionOrphanResourceAutoDeletion = SettingDefinition{
+		DisplayName: "Orphan Resource Auto-Deletion",
+		Description: "This setting allows Longhorn to automatically delete orphan resources and their corresponding orphaned resources. \n\n" +
+			"Orphan resources located on nodes that are in down or unknown state will not be cleaned up automatically. \n\n" +
+			"List the enabled resource types in a semicolon-separated list. \n\n" +
+			"Available items are: \n\n" +
+			"- **replicaData**: replica data store \n\n" +
+			"- **engineInstance**: engine runtime instance \n\n" +
+			"- **replicaInstance**: replica runtime instance \n\n",
+		Category: SettingCategoryOrphan,
+		Type:     SettingTypeString,
+		Required: false,
+		ReadOnly: false,
+		Default:  "",
+	}
+
 	SettingDefinitionStorageNetwork = SettingDefinition{
 		DisplayName: "Storage Network",
 		Description: "Longhorn uses the storage network for in-cluster data traffic. Leave this blank to use the Kubernetes cluster network. \n\n" +
@@ -1592,6 +1612,14 @@ const (
 	CNIAnnotationNetworksStatus = CNIAnnotation("k8s.v1.cni.cncf.io/networks-status")
 )
 
+type OrphanResourceType string
+
+const (
+	OrphanResourceTypeReplicaData     = OrphanResourceType("replicaData")
+	OrphanResourceTypeEngineInstance  = OrphanResourceType("engineInstance")
+	OrphanResourceTypeReplicaInstance = OrphanResourceType("replicaInstance")
+)
+
 func ValidateSetting(name, value string) (err error) {
 	defer func() {
 		err = errors.Wrapf(err, "value %v of settings %v is invalid", value, name)
@@ -1771,6 +1799,32 @@ func UnmarshalNodeSelector(nodeSelectorSetting string) (map[string]string, error
 	return nodeSelector, nil
 }
 
+func UnmarshalOrphanResourceTypes(resourceTypesSetting string) (map[OrphanResourceType]bool, error) {
+	resourceTypes := map[OrphanResourceType]bool{
+		OrphanResourceTypeReplicaData:     false,
+		OrphanResourceTypeEngineInstance:  false,
+		OrphanResourceTypeReplicaInstance: false,
+	}
+
+	resourceTypesSetting = strings.Trim(resourceTypesSetting, " ")
+	invalidItems := make([]string, 0, len(resourceTypesSetting))
+	if resourceTypesSetting != "" {
+		resourceTypeList := strings.Split(resourceTypesSetting, ";")
+		for _, item := range resourceTypeList {
+			resourceType := OrphanResourceType(strings.Trim(item, " "))
+			if _, ok := resourceTypes[resourceType]; ok {
+				resourceTypes[resourceType] = true
+			} else {
+				invalidItems = append(invalidItems, item)
+			}
+		}
+	}
+	if len(invalidItems) > 0 {
+		return nil, fmt.Errorf("invalid orphan resource types: %s", strings.Join(invalidItems, ", "))
+	}
+	return resourceTypes, nil
+}
+
 // GetSettingDefinition gets the setting definition in `settingDefinitions` by the parameter `name`
 func GetSettingDefinition(name SettingName) (SettingDefinition, bool) {
 	settingDefinitionsLock.RLock()
@@ -1876,6 +1930,11 @@ func validateString(sName SettingName, definition SettingDefinition, value strin
 		if err := ValidateV2DataEngineLogFlags(value); err != nil {
 			return errors.Wrapf(err, "failed to validate v2 data engine log flags %v", value)
 		}
+
+	case SettingNameOrphanResourceAutoDeletion:
+		if _, err := UnmarshalOrphanResourceTypes(value); err != nil {
+			return errors.Wrapf(err, "the value of %v is invalid", sName)
+		}
 	}
 
 	return nil
diff --git a/types/types.go b/types/types.go
index 65f12b73d6..8c75590bef 100644
--- a/types/types.go
+++ b/types/types.go
@@ -120,6 +120,8 @@ const (
 	DefaultBackupTargetName = "default"
 
 	LonghornNodeKey     = "longhornnode"
+	LonghornEngineKey   = "longhornengine"
+	LonghornReplicaKey  = "longhornreplica"
 	LonghornDiskUUIDKey = "longhorndiskuuid"
 
 	NodeCreateDefaultDiskLabelKey             = "node.longhorn.io/create-default-disk"
@@ -654,7 +656,25 @@ func GetOrphanLabelsForOrphanedDirectory(nodeID, diskUUID string) map[string]str
 	labels := GetBaseLabelsForSystemManagedComponent()
 	labels[GetLonghornLabelComponentKey()] = LonghornLabelOrphan
 	labels[LonghornNodeKey] = nodeID
-	labels[GetLonghornLabelKey(LonghornLabelOrphanType)] = string(longhorn.OrphanTypeReplica)
+	labels[GetLonghornLabelKey(LonghornLabelOrphanType)] = string(longhorn.OrphanTypeReplicaData)
+	return labels
+}
+
+func GetOrphanLabelsForOrphanedEngineInstance(nodeID, engineName string) map[string]string {
+	labels := GetBaseLabelsForSystemManagedComponent()
+	labels[GetLonghornLabelComponentKey()] = LonghornLabelOrphan
+	labels[LonghornNodeKey] = nodeID
+	labels[LonghornEngineKey] = engineName
+	labels[GetLonghornLabelKey(LonghornLabelOrphanType)] = string(longhorn.OrphanTypeEngineInstance)
+	return labels
+}
+
+func GetOrphanLabelsForOrphanedReplicaInstance(nodeID, replicaName string) map[string]string {
+	labels := GetBaseLabelsForSystemManagedComponent()
+	labels[GetLonghornLabelComponentKey()] = LonghornLabelOrphan
+	labels[LonghornNodeKey] = nodeID
+	labels[LonghornReplicaKey] = replicaName
+	labels[GetLonghornLabelKey(LonghornLabelOrphanType)] = string(longhorn.OrphanTypeReplicaInstance)
 	return labels
 }
 
@@ -726,6 +746,10 @@ func GetOrphanChecksumNameForOrphanedDataStore(nodeID, diskName, diskPath, diskU
 	return orphanPrefix + util.GetStringChecksumSHA256(strings.TrimSpace(fmt.Sprintf("%s-%s-%s-%s-%s", nodeID, diskName, diskPath, diskUUID, dataStore)))
 }
 
+func GetOrphanChecksumNameForOrphanedInstance(instanceName, nodeID, dataEngine string) string {
+	return orphanPrefix + util.GetStringChecksumSHA256(strings.TrimSpace(fmt.Sprintf("%s-%s-%s", instanceName, nodeID, dataEngine)))
+}
+
 func GetShareManagerPodNameFromShareManagerName(smName string) string {
 	return shareManagerPrefix + smName
 }
diff --git a/upgrade/v18xto190/upgrade.go b/upgrade/v18xto190/upgrade.go
index bdb045efce..17e71daa0c 100644
--- a/upgrade/v18xto190/upgrade.go
+++ b/upgrade/v18xto190/upgrade.go
@@ -1,9 +1,12 @@
 package v18xto190
 
 import (
+	"strings"
+
 	"github.com/pkg/errors"
 
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	clientset "k8s.io/client-go/kubernetes"
 
 	"github.com/longhorn/longhorn-manager/types"
@@ -32,6 +35,14 @@ func UpgradeResources(namespace string, lhClient *lhclientset.Clientset, kubeCli
 		return errors.New("resourceMaps cannot be nil")
 	}
 
+	if err := updateCRs(namespace, lhClient, kubeClient, resourceMaps); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func updateCRs(namespace string, lhClient *lhclientset.Clientset, kubeClient *clientset.Clientset, resourceMaps map[string]interface{}) (err error) {
 	// From v1.9.0, the v1beta1 API is deprecated, and v1beta2 is the storage version. Load all resource and write back into v1beta2.
 
 	updates := map[string]listAndUpdateFunc{
@@ -53,7 +64,7 @@ func UpgradeResources(namespace string, lhClient *lhclientset.Clientset, kubeCli
 	}
 	for resourceKind, listUpdateFunc := range updates {
 		if err := listUpdateFunc(namespace, lhClient, resourceMaps); err != nil {
-			return errors.Wrapf(err, "upgrade from v1.8.x to v1.9.0: failed to list all existing Longhorn %s during the %s upgrade", resourceKind, resourceKind)
+			return errors.Wrapf(err, upgradeLogPrefix+"failed to list all existing Longhorn %s during the %s upgrade", resourceKind, resourceKind)
 		}
 	}
 
@@ -61,6 +72,10 @@ func UpgradeResources(namespace string, lhClient *lhclientset.Clientset, kubeCli
 		return err
 	}
 
+	if err := upgradeSettings(namespace, lhClient, resourceMaps); err != nil {
+		return err
+	}
+
 	return nil
 }
 
@@ -85,3 +100,53 @@ func upgradeVolumes(namespace string, lhClient *lhclientset.Clientset, resourceM
 
 	return nil
 }
+
+func upgradeSettings(namespace string, lhClient *lhclientset.Clientset, resourceMaps map[string]interface{}) (err error) {
+	defer func() {
+		err = errors.Wrapf(err, upgradeLogPrefix+"upgrade settings failed")
+	}()
+
+	settingsMap, err := upgradeutil.ListAndUpdateSettingsInProvidedCache(namespace, lhClient, resourceMaps)
+	if err != nil {
+		if apierrors.IsNotFound(err) {
+			return nil
+		}
+		return errors.Wrapf(err, "failed to list all existing Longhorn settings during the settings upgrade")
+	}
+
+	if err := updateOrphanResourceAutoDeletionSetting(settingsMap); err != nil {
+		return errors.Wrapf(err, "failed to update orphan resource auto-deletion settings")
+	}
+
+	return nil
+}
+
+func updateOrphanResourceAutoDeletionSetting(settingsMap map[string]*longhorn.Setting) (err error) {
+	oldSetting, exist := settingsMap[string(types.SettingNameOrphanAutoDeletion)]
+	if !exist {
+		return nil
+	}
+	newSetting, exist := settingsMap[string(types.SettingNameOrphanResourceAutoDeletion)]
+	if !exist {
+		newSetting = &longhorn.Setting{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: string(types.SettingNameOrphanResourceAutoDeletion),
+			},
+		}
+		settingsMap[string(types.SettingNameOrphanResourceAutoDeletion)] = newSetting
+	}
+	resourceTypes, err := types.UnmarshalOrphanResourceTypes(newSetting.Value)
+	if err != nil {
+		return err
+	}
+
+	resourceTypes[types.OrphanResourceTypeReplicaData] = oldSetting.Value == "true"
+	enabledResourceType := make([]string, 0, len(resourceTypes))
+	for rt, enabled := range resourceTypes {
+		if enabled {
+			enabledResourceType = append(enabledResourceType, string(rt))
+		}
+	}
+	newSetting.Value = strings.Join(enabledResourceType, ";")
+	return nil
+}
diff --git a/webhook/resources/orphan/mutator.go b/webhook/resources/orphan/mutator.go
index ab562354d9..5490a7cb5a 100644
--- a/webhook/resources/orphan/mutator.go
+++ b/webhook/resources/orphan/mutator.go
@@ -56,8 +56,13 @@ func (o *orphanMutator) Create(request *admission.Request, newObj runtime.Object
 
 	// Add labels according to the orphan type
 	var longhornLabels map[string]string
-	if orphan.Spec.Type == longhorn.OrphanTypeReplica {
+	switch orphan.Spec.Type {
+	case longhorn.OrphanTypeReplicaData:
 		longhornLabels = types.GetOrphanLabelsForOrphanedDirectory(orphan.Spec.NodeID, orphan.Spec.Parameters[longhorn.OrphanDiskUUID])
+	case longhorn.OrphanTypeEngineInstance:
+		longhornLabels = types.GetOrphanLabelsForOrphanedEngineInstance(orphan.Spec.NodeID, orphan.Spec.Parameters[longhorn.OrphanInstanceName])
+	case longhorn.OrphanTypeReplicaInstance:
+		longhornLabels = types.GetOrphanLabelsForOrphanedReplicaInstance(orphan.Spec.NodeID, orphan.Spec.Parameters[longhorn.OrphanInstanceName])
 	}
 	if longhornLabels == nil {
 		return nil, werror.NewInvalidError("invalid orphan labels", "")
diff --git a/webhook/resources/orphan/validator.go b/webhook/resources/orphan/validator.go
index 7ebae1efef..b339e442db 100644
--- a/webhook/resources/orphan/validator.go
+++ b/webhook/resources/orphan/validator.go
@@ -45,8 +45,10 @@ func (o *orphanValidator) Create(request *admission.Request, newObj runtime.Obje
 
 	var err error
 	switch orphan.Spec.Type {
-	case longhorn.OrphanTypeReplica:
-		err = checkOrphanForReplicaDataStore(orphan)
+	case longhorn.OrphanTypeReplicaData:
+		err = checkOrphanForReplicaData(orphan)
+	case longhorn.OrphanTypeEngineInstance, longhorn.OrphanTypeReplicaInstance:
+		err = checkOrphanForInstance(orphan)
 	default:
 		return werror.NewInvalidError(fmt.Sprintf("unknown orphan type %v for orphan %v", orphan.Spec.Type, orphan.Name), "")
 	}
@@ -71,14 +73,17 @@ func (o *orphanValidator) Update(request *admission.Request, oldObj runtime.Obje
 }
 
 func checkOrphanParameters(orphan *longhorn.Orphan) error {
-	if orphan.Spec.Type == longhorn.OrphanTypeReplica {
-		return checkOrphanForReplicaDataStore(orphan)
+	switch orphan.Spec.Type {
+	case longhorn.OrphanTypeReplicaData:
+		return checkOrphanForReplicaData(orphan)
+	case longhorn.OrphanTypeEngineInstance, longhorn.OrphanTypeReplicaInstance:
+		return checkOrphanForInstance(orphan)
 	}
 
 	return werror.NewInvalidError(fmt.Sprintf("unknown orphan type %v for orphan %v", orphan.Spec.Type, orphan.Name), "")
 }
 
-func checkOrphanForReplicaDataStore(orphan *longhorn.Orphan) error {
+func checkOrphanForReplicaData(orphan *longhorn.Orphan) error {
 	params := []string{
 		longhorn.OrphanDataName,
 		longhorn.OrphanDiskName,
@@ -96,3 +101,29 @@ func checkOrphanForReplicaDataStore(orphan *longhorn.Orphan) error {
 
 	return nil
 }
+
+func checkOrphanForInstance(orphan *longhorn.Orphan) error {
+	params := []string{
+		longhorn.OrphanInstanceName,
+		longhorn.OrphanInstanceManager,
+	}
+
+	for _, param := range params {
+		val, ok := orphan.Spec.Parameters[param]
+		if !ok {
+			return fmt.Errorf("parameter %v for orphan %v is missing", param, orphan.Name)
+		}
+		if val == "" {
+			return fmt.Errorf("parameter %v for orphan %v is empty", param, orphan.Name)
+		}
+	}
+
+	switch orphan.Spec.DataEngine {
+	case longhorn.DataEngineTypeV1, longhorn.DataEngineTypeV2:
+		break
+	default:
+		return fmt.Errorf("invalid data engine type %v for orphan %v", orphan.Spec.DataEngine, orphan.Name)
+	}
+
+	return nil
+}


Reviews:
user: login: derekbit
body: I will continue the review later.
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-17T23:38:32Z
user: login: c3y1huang
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-18T02:37:20Z
user: login: COLDTURNIP
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-18T03:01:20Z
user: login: COLDTURNIP
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-18T03:16:31Z
user: login: c3y1huang
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-18T04:36:32Z
user: login: COLDTURNIP
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-18T08:41:38Z
user: login: COLDTURNIP
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-18T08:53:58Z
user: login: PhanLe1010
state: COMMENTED
author_association: NONE
submitted_at: 2025-04-18T23:26:45Z
user: login: PhanLe1010
state: COMMENTED
author_association: NONE
submitted_at: 2025-04-18T23:28:12Z
user: login: PhanLe1010
state: COMMENTED
author_association: NONE
submitted_at: 2025-04-18T23:43:38Z
user: login: PhanLe1010
state: COMMENTED
author_association: NONE
submitted_at: 2025-04-18T23:46:18Z
user: login: PhanLe1010
state: COMMENTED
author_association: NONE
submitted_at: 2025-04-19T00:20:04Z
user: login: PhanLe1010
state: COMMENTED
author_association: NONE
submitted_at: 2025-04-19T00:31:17Z
user: login: derekbit
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-20T05:28:14Z
user: login: COLDTURNIP
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-21T01:21:25Z
user: login: derekbit
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-21T01:26:37Z
user: login: COLDTURNIP
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-21T01:27:13Z
user: login: derekbit
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-21T01:34:46Z
user: login: COLDTURNIP
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-21T01:40:31Z
user: login: derekbit
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-21T01:46:02Z
user: login: COLDTURNIP
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-21T02:48:12Z
user: login: COLDTURNIP
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-21T10:49:55Z
user: login: PhanLe1010
state: COMMENTED
author_association: NONE
submitted_at: 2025-04-22T01:45:33Z
user: login: PhanLe1010
state: COMMENTED
author_association: NONE
submitted_at: 2025-04-22T01:52:22Z
user: login: PhanLe1010
state: COMMENTED
author_association: NONE
submitted_at: 2025-04-22T01:59:12Z
user: login: PhanLe1010
state: COMMENTED
author_association: NONE
submitted_at: 2025-04-22T02:08:42Z
user: login: PhanLe1010
body: In general, LGTM except for some NITs above
state: COMMENTED
author_association: NONE
submitted_at: 2025-04-22T02:11:47Z
user: login: COLDTURNIP
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-22T02:40:48Z
user: login: derekbit
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-22T04:08:26Z
user: login: derekbit
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-22T04:30:41Z
user: login: derekbit
body: In general, LGTM. Suggest improving the functions and variables naming and breaking down the longer functions into smaller ones for better readability.
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-22T07:58:32Z
user: login: COLDTURNIP
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-22T10:28:56Z
user: login: COLDTURNIP
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-22T10:37:13Z
user: login: COLDTURNIP
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-22T10:46:01Z
user: login: COLDTURNIP
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-22T10:53:05Z
user: login: COLDTURNIP
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-22T11:02:49Z
user: login: derekbit
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-22T11:35:18Z
user: login: COLDTURNIP
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-22T16:17:11Z
user: login: COLDTURNIP
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-22T16:19:12Z
user: login: COLDTURNIP
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-22T16:22:51Z
user: login: COLDTURNIP
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-22T16:24:10Z
user: login: COLDTURNIP
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-22T16:25:56Z
user: login: COLDTURNIP
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-22T16:26:17Z
user: login: derekbit
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-23T00:40:57Z
user: login: derekbit
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-23T00:49:55Z
user: login: derekbit
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-23T03:07:50Z
user: login: COLDTURNIP
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-23T03:17:06Z
user: login: COLDTURNIP
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-23T03:41:15Z
user: login: derekbit
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-23T04:07:18Z
user: login: COLDTURNIP
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-23T04:52:20Z
user: login: derekbit
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-23T10:20:47Z
user: login: sushant-suse
body: Some suggestions for your consideration.
Thanks.
state: CHANGES_REQUESTED
author_association: CONTRIBUTOR
submitted_at: 2025-04-23T12:18:39Z
user: login: COLDTURNIP
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-23T13:43:34Z
user: login: sushant-suse
body: LGTM.
Thanks.
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-04-23T14:47:15Z
user: login: derekbit
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-23T14:58:38Z
user: login: derekbit
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-23T15:00:50Z
user: login: derekbit
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-23T15:07:01Z
user: login: coderabbitai[bot]
body: **Actionable comments posted: 8**

<details>
<summary>🔭 Outside diff range comments (1)</summary><blockquote>

<details>
<summary>webhook/resources/orphan/validator.go (1)</summary><blockquote>

`86-103`: _🛠️ Refactor suggestion_

**Also reject empty parameter values for replica‑data orphans**

The current check only verifies the key exists, not that it is non‑empty.  
An orphan with `diskUUID=""` passes validation but is unusable later.

```diff
-    _, ok := orphan.Spec.Parameters[param]
-    if !ok {
+    val, ok := orphan.Spec.Parameters[param]
+    if !ok || strings.TrimSpace(val) == "" {
         return fmt.Errorf("parameter %v for orphan %v is missing", param, orphan.Name)
     }
```

</blockquote></details>

</blockquote></details>

<details>
<summary>🧹 Nitpick comments (8)</summary><blockquote>

<details>
<summary>k8s/pkg/apis/longhorn/v1beta2/instancemanager.go (1)</summary><blockquote>

`58-64`: **Consider adding API validation for `InstanceState`**  
While adding the new `InstanceStateTerminated` constant is correct, the `InstanceState` type currently has no OpenAPI enum validation. To prevent invalid states from being persisted, annotate the type with a `+kubebuilder:validation:Enum` that includes the new `"terminated"` value.  

```diff
+// +kubebuilder:validation:Enum=running;stopped;error;starting;stopping;unknown;terminated
 type InstanceState string

 const (
 	InstanceStateRunning    = InstanceState("running")
 	InstanceStateStopped    = InstanceState("stopped")
 	InstanceStateError      = InstanceState("error")
 	InstanceStateStarting   = InstanceState("starting")
 	InstanceStateStopping   = InstanceState("stopping")
 	InstanceStateUnknown    = InstanceState("unknown")
 	InstanceStateTerminated = InstanceState("terminated")
 )
```

</blockquote></details>
<details>
<summary>types/setting.go (1)</summary><blockquote>

`1802-1826`: **Implemented parser for orphan resource types setting.**

The `UnmarshalOrphanResourceTypes` function properly:
- Initializes all resource types to false by default
- Parses semicolon-separated resource type list
- Validates each entry against known resource types
- Returns clear error messages for invalid entries

This function could benefit from unit tests to verify proper handling of various input formats.

Consider adding unit tests for this function to ensure it correctly handles valid inputs, empty strings, and invalid resource types.

</blockquote></details>
<details>
<summary>types/types.go (2)</summary><blockquote>

`122-126`: **Consider aligning newly‑added label keys with existing naming helpers**

`LonghornEngineKey` and `LonghornReplicaKey` follow the same “raw‑string” style as the older `LonghornNodeKey`, which is perfectly valid.  
However, the rest of the codebase has gradually been moving toward `GetLonghornLabelKey(...)` to automatically prepend `longhorn.io/`.  Using that helper here would:

* keep the external label surface consistent (`kubectl get … -l longhorn.io/longhornengine=...`)
* avoid future duplication if we ever need both the raw and fully–qualified forms.

If you decide to keep the raw form, please leave a short comment explaining why these three keys are intentionally un‑prefixed so future contributors don’t “fix” them back.

---

`663-670`: **Duplication with GetOrphanLabelsForOrphanedReplicaInstance**

`GetOrphanLabelsForOrphanedEngineInstance`/`...ReplicaInstance` are almost identical except for the instance‑type‑specific key & orphan‑type constant.  
A tiny helper will remove duplication and ensure both branches always evolve together:

```go
func getOrphanInstanceLabels(nodeID, name, key, orphanType string) map[string]string {
    labels := GetBaseLabelsForSystemManagedComponent()
    labels[GetLonghornLabelComponentKey()] = LonghornLabelOrphan
    labels[LonghornNodeKey] = nodeID
    labels[key] = name
    labels[GetLonghornLabelKey(LonghornLabelOrphanType)] = orphanType
    return labels
}
```

</blockquote></details>
<details>
<summary>upgrade/v18xto190/upgrade.go (1)</summary><blockquote>

`50-65`: **Duplicate cache walks for Settings**

Settings are listed once in the generic `updates` loop and again in `upgradeSettings`.  
The second `ListAndUpdateSettingsInProvidedCache` re‑hits the API server and cache for no additional benefit because only `updateOrphanResourceAutoDeletionSetting` mutates data.  
You can avoid the extra round‑trip by:

* moving `updateOrphanResourceAutoDeletionSetting` directly after the first settings fetch, or
* passing the already‑retrieved `settingsMap` into `upgradeSettings`.

This speeds up upgrades and reduces pressure on the apiserver.

</blockquote></details>
<details>
<summary>webhook/resources/orphan/validator.go (1)</summary><blockquote>

`121-126`: **Unnecessary `break` statement & missing default case**

`break` after the last (and only) statement inside a `case` is redundant, and static analysis already flags it.  
Removing it makes the switch more idiomatic:

```diff
 case longhorn.DataEngineTypeV1, longhorn.DataEngineTypeV2:
-    break
+    // ok
```

You may also want a `case ""` to allow empty (=default) DataEngine.

<details>
<summary>🧰 Tools</summary>

<details>
<summary>🪛 GitHub Check: CodeFactor</summary>

[notice] 123-123: webhook/resources/orphan/validator.go#L123
Useless break in case clause. (useless-break)

---

[notice] 123-123: webhook/resources/orphan/validator.go#L123
Omit unnecessary break at the end of case clause. (unnecessary-stmt)

</details>

</details>

</blockquote></details>
<details>
<summary>controller/instance_manager_controller.go (2)</summary><blockquote>

`630-635`: **`deleteOrphans()` is invoked twice in the same reconciliation loop**  

`handlePod()` already calls `deleteOrphans()` when the pod is gone.  
Later, `syncInstanceManager()` unconditionally calls `syncOrphans()` which
again invokes `deleteOrphans()`.  This produces duplicate API traffic,
log noise, and races when two goroutines attempt to delete the same CR.

Guard the second call, or make `handlePod()` set a flag in‐memory so the
outer call can be skipped.

---

`1726-1752`: **Duplicate helper implementations for engine/replica scheduling**  

`isEngineScheduledOnCurrentNode()` and `isReplicaScheduledOnCurrentNode()` are almost identical.
Abstract the common logic to avoid divergence bugs:

```go
func (imc *InstanceManagerController) isInstanceCRScheduled(node, name string,
      get func(string)(spec *longhorn.InstanceSpec,status *longhorn.InstanceStatus,error)) (bool,error)
```

Then call it from both helpers.

</blockquote></details>

</blockquote></details>

<details>
<summary>📜 Review details</summary>

**Configuration used: .coderabbit.yaml**
**Review profile: CHILL**
**Plan: Pro**

<details>
<summary>📥 Commits</summary>

Reviewing files that changed from the base of the PR and between f190847e19e0ecc8909a65c95a0ba44e3f6a1842 and d62e66490e251433f88400da7ea9e3d3e77583ef.

</details>

<details>
<summary>📒 Files selected for processing (16)</summary>

* `api/model.go` (1 hunks)
* `controller/instance_manager_controller.go` (12 hunks)
* `controller/node_controller.go` (4 hunks)
* `controller/node_controller_test.go` (1 hunks)
* `controller/orphan_controller.go` (7 hunks)
* `controller/setting_controller.go` (3 hunks)
* `datastore/longhorn.go` (2 hunks)
* `k8s/crds.yaml` (1 hunks)
* `k8s/pkg/apis/longhorn/v1beta2/instancemanager.go` (1 hunks)
* `k8s/pkg/apis/longhorn/v1beta2/orphan.go` (3 hunks)
* `k8s/pkg/client/applyconfiguration/longhorn/v1beta2/orphanspec.go` (2 hunks)
* `types/setting.go` (7 hunks)
* `types/types.go` (3 hunks)
* `upgrade/v18xto190/upgrade.go` (4 hunks)
* `webhook/resources/orphan/mutator.go` (1 hunks)
* `webhook/resources/orphan/validator.go` (3 hunks)

</details>

<details>
<summary>🧰 Additional context used</summary>

<details>
<summary>🧠 Learnings (1)</summary>

<details>
<summary>upgrade/v18xto190/upgrade.go (1)</summary>

```
Learnt from: mantissahz
PR: longhorn/longhorn-manager#3648
File: upgrade/v18xto190/upgrade.go:67-86
Timestamp: 2025-04-21T06:56:13.403Z
Learning: In `upgrade/v18xto190/upgrade.go`, changes made to volumes in the `upgradeVolumes` function after calling `ListAndUpdateVolumesInProvidedCache` are expected to be persisted by the caller, not within the function itself.
```

</details>

</details><details>
<summary>🧬 Code Graph Analysis (8)</summary>

<details>
<summary>controller/node_controller_test.go (1)</summary><blockquote>

<details>
<summary>k8s/pkg/apis/longhorn/v1beta2/orphan.go (1)</summary>

* `OrphanTypeReplicaData` (8-8)

</details>

</blockquote></details>
<details>
<summary>k8s/pkg/client/applyconfiguration/longhorn/v1beta2/orphanspec.go (2)</summary><blockquote>

<details>
<summary>k8s/pkg/apis/longhorn/v1beta2/orphan.go (1)</summary>

* `OrphanType` (5-5)

</details>
<details>
<summary>k8s/pkg/apis/longhorn/v1beta2/volume.go (1)</summary>

* `DataEngineType` (185-185)

</details>

</blockquote></details>
<details>
<summary>webhook/resources/orphan/mutator.go (2)</summary><blockquote>

<details>
<summary>k8s/pkg/apis/longhorn/v1beta2/orphan.go (5)</summary>

* `OrphanTypeReplicaData` (8-8)
* `OrphanDiskUUID` (31-31)
* `OrphanTypeEngineInstance` (9-9)
* `OrphanInstanceName` (26-26)
* `OrphanTypeReplicaInstance` (10-10)

</details>
<details>
<summary>types/types.go (3)</summary>

* `GetOrphanLabelsForOrphanedDirectory` (655-661)
* `GetOrphanLabelsForOrphanedEngineInstance` (663-670)
* `GetOrphanLabelsForOrphanedReplicaInstance` (672-679)

</details>

</blockquote></details>
<details>
<summary>controller/node_controller.go (2)</summary><blockquote>

<details>
<summary>types/setting.go (3)</summary>

* `SettingName` (51-51)
* `SettingNameOrphanResourceAutoDeletion` (103-103)
* `OrphanResourceTypeReplicaData` (1618-1618)

</details>
<details>
<summary>k8s/pkg/apis/longhorn/v1beta2/orphan.go (1)</summary>

* `OrphanTypeReplicaData` (8-8)

</details>

</blockquote></details>
<details>
<summary>types/types.go (2)</summary><blockquote>

<details>
<summary>k8s/pkg/apis/longhorn/v1beta2/orphan.go (3)</summary>

* `OrphanTypeReplicaData` (8-8)
* `OrphanTypeEngineInstance` (9-9)
* `OrphanTypeReplicaInstance` (10-10)

</details>
<details>
<summary>util/util.go (1)</summary>

* `GetStringChecksumSHA256` (410-412)

</details>

</blockquote></details>
<details>
<summary>webhook/resources/orphan/validator.go (3)</summary><blockquote>

<details>
<summary>k8s/pkg/apis/longhorn/v1beta2/orphan.go (6)</summary>

* `OrphanTypeReplicaData` (8-8)
* `OrphanTypeEngineInstance` (9-9)
* `OrphanTypeReplicaInstance` (10-10)
* `Orphan` (73-79)
* `OrphanInstanceName` (26-26)
* `OrphanInstanceManager` (27-27)

</details>
<details>
<summary>api/model.go (1)</summary>

* `Orphan` (592-596)

</details>
<details>
<summary>k8s/pkg/apis/longhorn/v1beta2/volume.go (2)</summary>

* `DataEngineTypeV1` (188-188)
* `DataEngineTypeV2` (189-189)

</details>

</blockquote></details>
<details>
<summary>k8s/pkg/apis/longhorn/v1beta2/orphan.go (1)</summary><blockquote>

<details>
<summary>k8s/pkg/apis/longhorn/v1beta2/volume.go (1)</summary>

* `DataEngineType` (185-185)

</details>

</blockquote></details>
<details>
<summary>controller/setting_controller.go (1)</summary><blockquote>

<details>
<summary>types/setting.go (3)</summary>

* `SettingNameOrphanResourceAutoDeletion` (103-103)
* `OrphanResourceTypeReplicaData` (1618-1618)
* `SettingNameOrphanAutoDeletion` (102-102)

</details>

</blockquote></details>

</details><details>
<summary>🪛 GitHub Check: CodeFactor</summary>

<details>
<summary>webhook/resources/orphan/validator.go</summary>

[notice] 123-123: webhook/resources/orphan/validator.go#L123
Useless break in case clause. (useless-break)

---

[notice] 123-123: webhook/resources/orphan/validator.go#L123
Omit unnecessary break at the end of case clause. (unnecessary-stmt)

</details>

</details>

</details>

<details>
<summary>⏰ Context from checks skipped due to timeout of 90000ms (1)</summary>

* GitHub Check: Summary

</details>

<details>
<summary>🔇 Additional comments (24)</summary><blockquote>

<details>
<summary>controller/node_controller_test.go (1)</summary>

`43-43`: **Test fixture updated to use the new orphan type.**  
The `DefaultOrphanTestNode1` now uses `OrphanTypeReplicaData` to align with the refactored orphan types. Under the hood this alias still maps to `"replica"`, so existing cleanup logic should remain valid.

</details>
<details>
<summary>api/model.go (1)</summary>

`2403-2403`: **Addition of DataEngine field aligns with expanded orphan types**

The addition of the `DataEngine` field to the `OrphanSpec` struct in the `toOrphanResource` function ensures that the API representation includes the data engine information from Orphan objects. This change properly supports the expanded orphan type handling introduced in this PR, which now distinguishes between replica data, engine instance, and replica instance orphans.

</details>
<details>
<summary>webhook/resources/orphan/mutator.go (1)</summary>

`59-66`: **Refined orphan type handling with specific labeling functions**

This change replaces a single if-condition with a comprehensive switch statement that handles the three distinct orphan types: replica data, engine instance, and replica instance. Each type now uses a dedicated labeling function with appropriate parameters. This approach provides better structure and extensibility for orphan management.

</details>
<details>
<summary>k8s/pkg/client/applyconfiguration/longhorn/v1beta2/orphanspec.go (2)</summary>

`30-30`: **Added DataEngine field to support orphan instance types**

The addition of the `DataEngine` field to the `OrphanSpecApplyConfiguration` struct is necessary to support the new orphan type system, enabling the association of orphans with specific data engine versions.

---

`56-62`: **Added builder method for fluent API support**

The `WithDataEngine` method provides a fluent builder pattern for setting the new `DataEngine` field, maintaining consistency with the other fields in the struct and allowing for clean, chainable configuration.

</details>
<details>
<summary>controller/setting_controller.go (2)</summary>

`903-927`: **Synchronized new fine-grained setting with legacy setting**

This method provides backward compatibility by syncing values between the new structured orphan resource auto-deletion setting and the deprecated boolean setting. The implementation correctly extracts the replica data-specific setting and updates the legacy setting when they differ, ensuring a smooth transition for users.

---

`1618-1618`: **Added new setting to metrics collection**

Including the `OrphanResourceAutoDeletion` setting in the metrics collection map ensures it will be properly reported in usage statistics, providing visibility into how this feature is configured across deployments.

</details>
<details>
<summary>datastore/longhorn.go (2)</summary>

`3526-3536`: **Well-designed method for orphan resource auto-deletion settings**

This method retrieves the `OrphanResourceAutoDeletion` setting as a map of orphan resource types to booleans, enabling granular control over auto-deletion of different orphan resource types. The implementation is clean and follows the pattern of other setting accessor methods in the codebase.

---

`5270-5285`: **Efficient method for filtering orphan resources by type**

This method appropriately filters orphans by node to only include `EngineInstance` and `ReplicaInstance` types, which aligns with the PR's focus on orphaned runtime instance cleanup. The read-only (RO) approach maintains efficiency by avoiding deep copies when the controller only needs to read the data.

</details>
<details>
<summary>controller/node_controller.go (3)</summary>

`197-197`: **Setting name update matches new granular orphan resource management approach.**

The code correctly updates to use the new setting name `SettingNameOrphanResourceAutoDeletion` in the responsibility check for settings.

---

`1331-1332`: **Updated orphan type name to align with new orphan resource classification.**

The condition now correctly uses `OrphanTypeReplicaData` instead of the deprecated `OrphanTypeReplica` to filter orphans consistently with the new orphan type definitions.

---

`1360-1364`: **Updated to use granular orphan resource auto-deletion settings.**

The code now uses `GetSettingOrphanResourceAutoDeletion()` which returns a map of resource types instead of a simple boolean, and correctly checks the setting for replica data orphans specifically.

</details>
<details>
<summary>k8s/pkg/apis/longhorn/v1beta2/orphan.go (4)</summary>

`8-10`: **Enhanced orphan type classification with distinct resource types.**

The revised orphan type system provides better separation of concerns with three distinct types:
- `OrphanTypeReplicaData` (formerly `OrphanTypeReplica`) for replica data storage
- New `OrphanTypeEngineInstance` for engine runtime instances
- New `OrphanTypeReplicaInstance` for replica runtime instances

This classification improves orphan resource management by enabling type-specific handling.

---

`16-16`: **Renamed condition type to better reflect instance existence checking.**

Changed from `OrphanConditionTypeRuntimeExists` to `OrphanConditionTypeInstanceExist` for improved terminology consistency in the instance-oriented context.

---

`26-28`: **Updated parameter names for consistent terminology.**

Renamed to use "Instance" consistently instead of "Runtime", and added `OrphanInstanceManager` to track the instance manager responsible for an orphaned instance.

---

`46-49`: **Added DataEngine field to support engine versioning in orphan specs.**

The addition of the `DataEngine` field with enum validation allows tracking which data engine (v1 or v2) an orphaned instance belongs to, which is essential for proper cleanup operations.

</details>
<details>
<summary>types/setting.go (5)</summary>

`103-103`: **Added new setting for granular orphan resource auto-deletion control.**

The new `SettingNameOrphanResourceAutoDeletion` setting provides a more flexible approach to orphan cleanup.

---

`1062-1067`: **Properly deprecated old orphan auto-deletion setting.**

The old setting is now marked as read-only, and its description clearly states it's deprecated in favor of using the specific replica data type in the new setting.

---

`1070-1084`: **Well-documented new orphan resource auto-deletion setting.**

The new setting definition includes clear documentation explaining:
- Purpose of auto-deletion for orphan resources
- Limitations regarding nodes in down or unknown state
- Available resource types with descriptions

This provides good user guidance for this feature.

---

`1615-1621`: **Added orphan resource type definitions for typed settings.**

Defined the `OrphanResourceType` string type and corresponding constants that match the orphan types in the CRD, providing type safety for settings handling.

---

`1934-1937`: **Added validation for the new orphan resource auto-deletion setting.**

The validation integration ensures that the setting value is checked for validity during setting updates.

</details>
<details>
<summary>types/types.go (2)</summary>

`655-661`: **No disk UUID label on directory orphans any more?**

Changing the orphan type to `replica` → `replica-data` is correct, but the function silently **dropped** the `LonghornDiskUUIDKey` that was formerly set by this helper.  
Unless the field moved elsewhere, we will lose the ability to filter directory orphans by disk UUID via labels.  Please double‑check and either:

```diff
+   labels[LonghornDiskUUIDKey] = diskUUID
```

or intentionally document the removal.

---

`672-679`: **Missing validation of empty strings**

`nodeID` or `replicaName` being empty will still produce a label map, which then fails validation deeper in the pipeline.  
Consider early‑returning an error (or at least logging) if either parameter is empty.

</details>
<details>
<summary>controller/orphan_controller.go (1)</summary>

`321-329`: **`NodeID` vs `controllerID` comparison is fragile**

Comparing `orphan.Spec.NodeID` with `oc.controllerID` assumes the controller ID
is **always** the Longhorn node name. If future refactors change the ID format
(e.g. add manager pod UID) this check will erroneously delete data.  
Consider comparing against the *current node name* from datastore or adding a
helper `IsSameNode()` used consistently across controllers.

</details>

</blockquote></details>

</details>

<!-- This is an auto-generated comment by CodeRabbit for review status -->
state: COMMENTED
author_association: NONE
submitted_at: 2025-04-23T15:26:29Z
user: login: COLDTURNIP
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-24T01:16:18Z
user: login: derekbit
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-24T01:21:20Z
user: login: derekbit
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-24T01:26:18Z
user: login: derekbit
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-24T01:32:06Z
user: login: COLDTURNIP
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-24T04:31:41Z
user: login: COLDTURNIP
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-24T04:32:23Z
user: login: coderabbitai[bot]
state: COMMENTED
author_association: NONE
submitted_at: 2025-04-24T04:32:29Z
user: login: coderabbitai[bot]
state: COMMENTED
author_association: NONE
submitted_at: 2025-04-24T04:34:20Z
user: login: COLDTURNIP
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-24T04:58:16Z
user: login: coderabbitai[bot]
state: COMMENTED
author_association: NONE
submitted_at: 2025-04-24T05:00:01Z
user: login: derekbit
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-24T05:20:44Z
user: login: COLDTURNIP
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-24T06:36:36Z
user: login: coderabbitai[bot]
state: COMMENTED
author_association: NONE
submitted_at: 2025-04-24T06:37:33Z
user: login: COLDTURNIP
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-04-24T07:33:28Z
user: login: derekbit
body: LGTM
state: DISMISSED
author_association: MEMBER
submitted_at: 2025-04-24T07:35:20Z
user: login: derekbit
state: APPROVED
author_association: MEMBER
submitted_at: 2025-04-24T08:47:07Z

Review comments:
diff_hunk: @@ -5,12 +5,15 @@ import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 type OrphanType string
 
 const (
-	OrphanTypeReplica = OrphanType("replica")
+	OrphanTypeReplicaDataStore = OrphanType("replica")
+	OrphanTypeEngineInstance   = OrphanType("engine-instance")
+	OrphanTypeReplicaInstance  = OrphanType("replica-instance")
 )
 
 const (
-	OrphanConditionTypeError         = "Error"
-	OrphanConditionTypeDataCleanable = "DataCleanable"
+	OrphanConditionTypeError                = "Error"
+	OrphanConditionTypeDataCleanable        = "DataCleanable"
+	OrphanConditionTypeRuntimeInstanceState = "InstanceState"
path: k8s/pkg/apis/longhorn/v1beta2/orphan.go
user: login: derekbit
body: I think we can remove `Runtime`. `Instance` should be enough and is already used in longhorn-manager and other componenets.

```suggestion
	OrphanConditionTypeInstanceState = "InstanceState"
```
created_at: 2025-04-17T23:34:08Z
author_association: MEMBER
diff_hunk: @@ -20,6 +23,9 @@ const (
 )
 
 const (
+	OrphanInstanceName   = "RuntimeInstanceName"
path: k8s/pkg/apis/longhorn/v1beta2/orphan.go
user: login: derekbit
body: ```suggestion
	OrphanInstanceName   = "InstanceName"
```
created_at: 2025-04-17T23:34:21Z
author_association: MEMBER
diff_hunk: @@ -269,48 +303,187 @@ func (oc *OrphanController) isResponsibleFor(orphan *longhorn.Orphan) bool {
 	return isControllerResponsibleFor(oc.controllerID, oc.ds, orphan.Name, orphan.Spec.NodeID, orphan.Status.OwnerID)
 }
 
-func (oc *OrphanController) cleanupOrphanedData(orphan *longhorn.Orphan) (err error) {
+func (oc *OrphanController) cleanupOrphanedResource(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
 	log := getLoggerForOrphan(oc.logger, orphan)
 
 	defer func() {
 		if err == nil {
 			return
 		}
 
-		err = errors.Wrapf(err, "failed to delete orphan %v data", orphan.Name)
+		err = errors.Wrapf(err, "failed to delete %v orphan %v", orphan.Spec.Type, orphan.Name)
 		orphan.Status.Conditions = types.SetCondition(orphan.Status.Conditions,
 			longhorn.OrphanConditionTypeError, longhorn.ConditionStatusTrue, "", err.Error())
 	}()
 
 	// Make sure if the orphan nodeID and controller ID are same.
 	// If NO, just delete the orphan resource object and don't touch the data.
 	if orphan.Spec.NodeID != oc.controllerID {
-		log.Infof("Orphan nodeID %v is different from controllerID %v, so just delete the orphan resource object",
-			orphan.Name, oc.controllerID)
-		return nil
+		log.WithField("orphanType", orphan.Spec.Type).Infof("Orphan nodeID %v is on %v. Delete the orphan resource object.",
+			orphan.Name, orphan.Spec.NodeID)
+		return true, nil
 	}
 
-	if types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeDataCleanable).Status !=
-		longhorn.ConditionStatusTrue {
-		log.Infof("Only delete orphan %v resource object and do not delete the orphaned data", orphan.Name)
-		return nil
+	switch orphan.Spec.Type {
+	case longhorn.OrphanTypeEngineInstance:
+		isCleanupComplete, err = oc.cleanupOrphanedEngineInstance(orphan)
+	case longhorn.OrphanTypeReplicaInstance:
+		isCleanupComplete, err = oc.cleanupOrphanedReplicaInstance(orphan)
+	case longhorn.OrphanTypeReplicaDataStore:
+		if types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeDataCleanable).Status !=
+			longhorn.ConditionStatusTrue {
+			log.Infof("Only delete orphan %v resource object and do not delete the orphaned data store", orphan.Name)
+			return true, nil
+		}
+		err = oc.deleteOrphanedReplicaDataStore(orphan)
+		if err == nil || (err != nil && apierrors.IsNotFound(err)) {
+			isCleanupComplete = true
+			err = nil
+		}
+	default:
+		isCleanupComplete = false
+		err = fmt.Errorf("unknown orphan type %v of orphan %v", orphan.Spec.Type, orphan.Name)
 	}
 
-	switch orphan.Spec.Type {
-	case longhorn.OrphanTypeReplica:
-		err = oc.deleteOrphanedReplica(orphan)
+	return isCleanupComplete, err
+}
+
+func (oc *OrphanController) cleanupOrphanedEngineInstance(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
+	defer func() {
+		if err != nil {
+			err = errors.Wrapf(err, "cannot cleanup engine instance orphan %v", orphan.Name)
+		}
+	}()
+
+	instance, deType, err := oc.extractOrphanedRuntimeInfo(orphan)
path: controller/orphan_controller.go
user: login: derekbit
body: Apply to other similar changes.
```suggestion
	instance, deType, err := oc.extractOrphanedInstanceInfo(orphan)
```
created_at: 2025-04-17T23:34:53Z
author_association: MEMBER
diff_hunk: @@ -1052,14 +1055,30 @@ var (
 	SettingDefinitionOrphanAutoDeletion = SettingDefinition{
 		DisplayName: "Orphan Auto-Deletion",
 		Description: "This setting allows Longhorn to delete the orphan resource and its corresponding orphaned data automatically. \n\n" +
-			"Orphan resources on down or unknown nodes will not be cleaned up automatically. \n\n",
+			"Orphan resources on down or unknown nodes will not be cleaned up automatically. \n\n" +
path: types/setting.go
user: login: derekbit
body: Mention the setting is deprecated and replaced by the new setting in the description.

cc @sushant-suse 
created_at: 2025-04-17T23:36:49Z
author_association: MEMBER
diff_hunk: @@ -612,6 +617,12 @@ func (imc *InstanceManagerController) handlePod(im *longhorn.InstanceManager) er
 		return err
 	}
 
+	if isPodDeletedOrNotRunning {
+		if err := imc.cleanupInstanceManagerOrphanRuntimeInstances(im); err != nil {
+			log.WithError(err).Warnf("Instance manager %v failed to cleanup runtime instance orphans for stopped instance manager pod", im.Name)
path: controller/instance_manager_controller.go
user: login: derekbit
body: nit:
```suggestion
			log.WithError(err).Warnf("Instance manager %v failed to cleanup orphaned instance for stopped instance manager pod", im.Name)
```
created_at: 2025-04-17T23:38:13Z
author_association: MEMBER
diff_hunk: @@ -93,6 +93,15 @@ func NewOrphanController(
 	}
 	oc.cacheSyncs = append(oc.cacheSyncs, ds.NodeInformer.HasSynced)
 
+	if _, err = ds.InstanceManagerInformer.AddEventHandlerWithResyncPeriod(cache.ResourceEventHandlerFuncs{
+		AddFunc:    func(cur interface{}) { oc.enqueueForInstanceManager(cur) },
+		UpdateFunc: func(old, cur interface{}) { oc.enqueueForInstanceManager(cur) },
+		DeleteFunc: func(cur interface{}) { oc.enqueueForInstanceManager(cur) },
+	}, 0); err != nil {
+		return nil, err
+	}
+	oc.cacheSyncs = append(oc.cacheSyncs, ds.NodeInformer.HasSynced)
path: controller/orphan_controller.go
user: login: c3y1huang
body: 
```suggestion
	oc.cacheSyncs = append(oc.cacheSyncs, ds.InstanceManagerInformer.HasSynced)
```
created_at: 2025-04-18T01:56:31Z
author_association: CONTRIBUTOR
diff_hunk: @@ -269,48 +303,187 @@ func (oc *OrphanController) isResponsibleFor(orphan *longhorn.Orphan) bool {
 	return isControllerResponsibleFor(oc.controllerID, oc.ds, orphan.Name, orphan.Spec.NodeID, orphan.Status.OwnerID)
 }
 
-func (oc *OrphanController) cleanupOrphanedData(orphan *longhorn.Orphan) (err error) {
+func (oc *OrphanController) cleanupOrphanedResource(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
 	log := getLoggerForOrphan(oc.logger, orphan)
 
 	defer func() {
 		if err == nil {
 			return
 		}
 
-		err = errors.Wrapf(err, "failed to delete orphan %v data", orphan.Name)
+		err = errors.Wrapf(err, "failed to delete %v orphan %v", orphan.Spec.Type, orphan.Name)
 		orphan.Status.Conditions = types.SetCondition(orphan.Status.Conditions,
 			longhorn.OrphanConditionTypeError, longhorn.ConditionStatusTrue, "", err.Error())
 	}()
 
 	// Make sure if the orphan nodeID and controller ID are same.
 	// If NO, just delete the orphan resource object and don't touch the data.
 	if orphan.Spec.NodeID != oc.controllerID {
-		log.Infof("Orphan nodeID %v is different from controllerID %v, so just delete the orphan resource object",
-			orphan.Name, oc.controllerID)
-		return nil
+		log.WithField("orphanType", orphan.Spec.Type).Infof("Orphan nodeID %v is on %v. Delete the orphan resource object.",
+			orphan.Name, orphan.Spec.NodeID)
path: controller/orphan_controller.go
user: login: c3y1huang
body: 
```suggestion
	// Make sure if the orphan nodeID and controller ID are same.
	// If NO, just delete the orphan resource object and don't touch the data.
	if orphan.Spec.NodeID != oc.controllerID {
		log.WithFields(logrus.Fields{
			"orphanType": orphan.Spec.Type,
			"orphanName": orphan.Name,
			"orphanNode": orphan.Spec.NodeID,
		}).Infof("Orphan does not belong to this controller. Skipping resource cleanup")
```
created_at: 2025-04-18T02:02:05Z
author_association: CONTRIBUTOR
diff_hunk: @@ -269,48 +303,187 @@ func (oc *OrphanController) isResponsibleFor(orphan *longhorn.Orphan) bool {
 	return isControllerResponsibleFor(oc.controllerID, oc.ds, orphan.Name, orphan.Spec.NodeID, orphan.Status.OwnerID)
 }
 
-func (oc *OrphanController) cleanupOrphanedData(orphan *longhorn.Orphan) (err error) {
+func (oc *OrphanController) cleanupOrphanedResource(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
 	log := getLoggerForOrphan(oc.logger, orphan)
 
 	defer func() {
 		if err == nil {
 			return
 		}
 
-		err = errors.Wrapf(err, "failed to delete orphan %v data", orphan.Name)
+		err = errors.Wrapf(err, "failed to delete %v orphan %v", orphan.Spec.Type, orphan.Name)
 		orphan.Status.Conditions = types.SetCondition(orphan.Status.Conditions,
 			longhorn.OrphanConditionTypeError, longhorn.ConditionStatusTrue, "", err.Error())
 	}()
 
 	// Make sure if the orphan nodeID and controller ID are same.
 	// If NO, just delete the orphan resource object and don't touch the data.
 	if orphan.Spec.NodeID != oc.controllerID {
-		log.Infof("Orphan nodeID %v is different from controllerID %v, so just delete the orphan resource object",
-			orphan.Name, oc.controllerID)
-		return nil
+		log.WithField("orphanType", orphan.Spec.Type).Infof("Orphan nodeID %v is on %v. Delete the orphan resource object.",
+			orphan.Name, orphan.Spec.NodeID)
+		return true, nil
 	}
 
-	if types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeDataCleanable).Status !=
-		longhorn.ConditionStatusTrue {
-		log.Infof("Only delete orphan %v resource object and do not delete the orphaned data", orphan.Name)
-		return nil
+	switch orphan.Spec.Type {
+	case longhorn.OrphanTypeEngineInstance:
+		isCleanupComplete, err = oc.cleanupOrphanedEngineInstance(orphan)
+	case longhorn.OrphanTypeReplicaInstance:
+		isCleanupComplete, err = oc.cleanupOrphanedReplicaInstance(orphan)
+	case longhorn.OrphanTypeReplicaDataStore:
+		if types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeDataCleanable).Status !=
+			longhorn.ConditionStatusTrue {
+			log.Infof("Only delete orphan %v resource object and do not delete the orphaned data store", orphan.Name)
+			return true, nil
+		}
+		err = oc.deleteOrphanedReplicaDataStore(orphan)
+		if err == nil || (err != nil && apierrors.IsNotFound(err)) {
+			isCleanupComplete = true
+			err = nil
+		}
+	default:
+		isCleanupComplete = false
+		err = fmt.Errorf("unknown orphan type %v of orphan %v", orphan.Spec.Type, orphan.Name)
 	}
 
-	switch orphan.Spec.Type {
-	case longhorn.OrphanTypeReplica:
-		err = oc.deleteOrphanedReplica(orphan)
+	return isCleanupComplete, err
+}
+
+func (oc *OrphanController) cleanupOrphanedEngineInstance(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
+	defer func() {
+		if err != nil {
+			err = errors.Wrapf(err, "cannot cleanup engine instance orphan %v", orphan.Name)
+		}
+	}()
+
+	instance, deType, err := oc.extractOrphanedRuntimeInfo(orphan)
+	if err != nil {
+		return false, err
+	}
+
+	var spec *longhorn.InstanceSpec
+	if engineCR, err := oc.ds.GetEngineRO(instance); err != nil {
+		if !apierrors.IsNotFound(err) {
+			return false, err
+		}
+		spec = nil
+	} else {
+		spec = &engineCR.Spec.InstanceSpec
+	}
+	return oc.cleanupOrphanedRuntimeInstance(orphan, instance, longhorn.InstanceManagerTypeEngine, deType, spec)
+}
+
+func (oc *OrphanController) cleanupOrphanedReplicaInstance(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
+	defer func() {
+		if err != nil {
+			err = errors.Wrapf(err, "cannot cleanup replica instance orphan %v", orphan.Name)
+		}
+	}()
+
+	instance, deType, err := oc.extractOrphanedRuntimeInfo(orphan)
+	if err != nil {
+		return false, err
+	}
+
+	var spec *longhorn.InstanceSpec
+	if replicaCR, err := oc.ds.GetReplicaRO(instance); err != nil {
+		if !apierrors.IsNotFound(err) {
+			return false, err
+		}
+		spec = nil
+	} else {
+		spec = &replicaCR.Spec.InstanceSpec
+	}
+	return oc.cleanupOrphanedRuntimeInstance(orphan, instance, longhorn.InstanceManagerTypeReplica, deType, spec)
+}
+
+func (oc *OrphanController) cleanupOrphanedRuntimeInstance(orphan *longhorn.Orphan, instance string, imType longhorn.InstanceManagerType, deType longhorn.DataEngineType, instanceCRSpec *longhorn.InstanceSpec) (isCleanupComplete bool, err error) {
+	if instanceCRSpec != nil && instanceCRSpec.NodeID == oc.controllerID {
+		// Instance is scheduled back to current node and is no longer considered orphaned.
+		// Finalize the orphan CR directly without terminating the instance process.
+		return true, nil
+	}
+
+	im, err := oc.ds.GetRunningInstanceManagerByNodeRO(oc.controllerID, deType)
+	if err != nil {
+		oc.logger.WithError(err).Infof("No running instance manager for node %v for deleting runtime instance orphan %v", oc.controllerID, orphan.Name)
+		return true, nil
+	}
+	imc, err := engineapi.NewInstanceManagerClient(im, false)
+	if err != nil {
+		return false, err
+	}
+	defer func() {
+		if closeErr := imc.Close(); closeErr != nil {
+			oc.logger.WithError(closeErr).Error("failed to close instance manager client")
+		}
+	}()
+
+	runtimeInstanceExist := types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeRuntimeInstanceState).Status == longhorn.ConditionStatusTrue
+	if runtimeInstanceExist {
+		if err = oc.deleteRuntimeInstance(imc, instance, imType, deType); err != nil {
+			return false, err
+		}
+	}
+	return oc.confirmOrphanRuntimeInstanceCleanup(imc, instance, imType, deType)
+}
+
+func (oc *OrphanController) extractOrphanedRuntimeInfo(orphan *longhorn.Orphan) (name string, dataEngineType longhorn.DataEngineType, err error) {
+	name, ok := orphan.Spec.Parameters[longhorn.OrphanInstanceName]
+	if !ok {
+		return "", "", fmt.Errorf("failed to get instance name for runtime instance orphan %v", orphan.Name)
+	}
+
+	dataEngineTypeStr, ok := orphan.Spec.Parameters[longhorn.OrphanDataEngineType]
+	if !ok {
+		return "", "", fmt.Errorf("failed to get data engine type for runtime instance orphan %v", orphan.Name)
+	}
+	switch dataEngineType = longhorn.DataEngineType(dataEngineTypeStr); dataEngineType {
+	case longhorn.DataEngineTypeV1, longhorn.DataEngineTypeV2:
+		// supported data engine type
 	default:
-		err = fmt.Errorf("unknown orphan type %v", orphan.Spec.Type)
+		return "", "", fmt.Errorf("unknown data engine type %v for runtime instance orphan %v", dataEngineTypeStr, orphan.Name)
path: controller/orphan_controller.go
user: login: c3y1huang
body: Can this be validated via webhook?
created_at: 2025-04-18T02:11:23Z
author_association: CONTRIBUTOR
diff_hunk: @@ -269,48 +303,187 @@ func (oc *OrphanController) isResponsibleFor(orphan *longhorn.Orphan) bool {
 	return isControllerResponsibleFor(oc.controllerID, oc.ds, orphan.Name, orphan.Spec.NodeID, orphan.Status.OwnerID)
 }
 
-func (oc *OrphanController) cleanupOrphanedData(orphan *longhorn.Orphan) (err error) {
+func (oc *OrphanController) cleanupOrphanedResource(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
 	log := getLoggerForOrphan(oc.logger, orphan)
 
 	defer func() {
 		if err == nil {
 			return
 		}
 
-		err = errors.Wrapf(err, "failed to delete orphan %v data", orphan.Name)
+		err = errors.Wrapf(err, "failed to delete %v orphan %v", orphan.Spec.Type, orphan.Name)
 		orphan.Status.Conditions = types.SetCondition(orphan.Status.Conditions,
 			longhorn.OrphanConditionTypeError, longhorn.ConditionStatusTrue, "", err.Error())
 	}()
 
 	// Make sure if the orphan nodeID and controller ID are same.
 	// If NO, just delete the orphan resource object and don't touch the data.
 	if orphan.Spec.NodeID != oc.controllerID {
-		log.Infof("Orphan nodeID %v is different from controllerID %v, so just delete the orphan resource object",
-			orphan.Name, oc.controllerID)
-		return nil
+		log.WithField("orphanType", orphan.Spec.Type).Infof("Orphan nodeID %v is on %v. Delete the orphan resource object.",
+			orphan.Name, orphan.Spec.NodeID)
+		return true, nil
 	}
 
-	if types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeDataCleanable).Status !=
-		longhorn.ConditionStatusTrue {
-		log.Infof("Only delete orphan %v resource object and do not delete the orphaned data", orphan.Name)
-		return nil
+	switch orphan.Spec.Type {
+	case longhorn.OrphanTypeEngineInstance:
+		isCleanupComplete, err = oc.cleanupOrphanedEngineInstance(orphan)
+	case longhorn.OrphanTypeReplicaInstance:
+		isCleanupComplete, err = oc.cleanupOrphanedReplicaInstance(orphan)
+	case longhorn.OrphanTypeReplicaDataStore:
+		if types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeDataCleanable).Status !=
+			longhorn.ConditionStatusTrue {
+			log.Infof("Only delete orphan %v resource object and do not delete the orphaned data store", orphan.Name)
+			return true, nil
+		}
+		err = oc.deleteOrphanedReplicaDataStore(orphan)
+		if err == nil || (err != nil && apierrors.IsNotFound(err)) {
+			isCleanupComplete = true
+			err = nil
+		}
+	default:
+		isCleanupComplete = false
+		err = fmt.Errorf("unknown orphan type %v of orphan %v", orphan.Spec.Type, orphan.Name)
 	}
 
-	switch orphan.Spec.Type {
-	case longhorn.OrphanTypeReplica:
-		err = oc.deleteOrphanedReplica(orphan)
+	return isCleanupComplete, err
+}
+
+func (oc *OrphanController) cleanupOrphanedEngineInstance(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
+	defer func() {
+		if err != nil {
+			err = errors.Wrapf(err, "cannot cleanup engine instance orphan %v", orphan.Name)
+		}
+	}()
+
+	instance, deType, err := oc.extractOrphanedRuntimeInfo(orphan)
+	if err != nil {
+		return false, err
+	}
+
+	var spec *longhorn.InstanceSpec
+	if engineCR, err := oc.ds.GetEngineRO(instance); err != nil {
+		if !apierrors.IsNotFound(err) {
+			return false, err
+		}
+		spec = nil
+	} else {
+		spec = &engineCR.Spec.InstanceSpec
+	}
+	return oc.cleanupOrphanedRuntimeInstance(orphan, instance, longhorn.InstanceManagerTypeEngine, deType, spec)
+}
+
+func (oc *OrphanController) cleanupOrphanedReplicaInstance(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
+	defer func() {
+		if err != nil {
+			err = errors.Wrapf(err, "cannot cleanup replica instance orphan %v", orphan.Name)
+		}
+	}()
+
+	instance, deType, err := oc.extractOrphanedRuntimeInfo(orphan)
+	if err != nil {
+		return false, err
+	}
+
+	var spec *longhorn.InstanceSpec
+	if replicaCR, err := oc.ds.GetReplicaRO(instance); err != nil {
+		if !apierrors.IsNotFound(err) {
+			return false, err
+		}
+		spec = nil
+	} else {
+		spec = &replicaCR.Spec.InstanceSpec
+	}
+	return oc.cleanupOrphanedRuntimeInstance(orphan, instance, longhorn.InstanceManagerTypeReplica, deType, spec)
+}
+
+func (oc *OrphanController) cleanupOrphanedRuntimeInstance(orphan *longhorn.Orphan, instance string, imType longhorn.InstanceManagerType, deType longhorn.DataEngineType, instanceCRSpec *longhorn.InstanceSpec) (isCleanupComplete bool, err error) {
+	if instanceCRSpec != nil && instanceCRSpec.NodeID == oc.controllerID {
+		// Instance is scheduled back to current node and is no longer considered orphaned.
+		// Finalize the orphan CR directly without terminating the instance process.
+		return true, nil
+	}
+
+	im, err := oc.ds.GetRunningInstanceManagerByNodeRO(oc.controllerID, deType)
+	if err != nil {
+		oc.logger.WithError(err).Infof("No running instance manager for node %v for deleting runtime instance orphan %v", oc.controllerID, orphan.Name)
+		return true, nil
+	}
+	imc, err := engineapi.NewInstanceManagerClient(im, false)
+	if err != nil {
+		return false, err
+	}
+	defer func() {
+		if closeErr := imc.Close(); closeErr != nil {
+			oc.logger.WithError(closeErr).Error("failed to close instance manager client")
+		}
+	}()
+
+	runtimeInstanceExist := types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeRuntimeInstanceState).Status == longhorn.ConditionStatusTrue
+	if runtimeInstanceExist {
+		if err = oc.deleteRuntimeInstance(imc, instance, imType, deType); err != nil {
+			return false, err
+		}
+	}
+	return oc.confirmOrphanRuntimeInstanceCleanup(imc, instance, imType, deType)
+}
+
+func (oc *OrphanController) extractOrphanedRuntimeInfo(orphan *longhorn.Orphan) (name string, dataEngineType longhorn.DataEngineType, err error) {
+	name, ok := orphan.Spec.Parameters[longhorn.OrphanInstanceName]
+	if !ok {
+		return "", "", fmt.Errorf("failed to get instance name for runtime instance orphan %v", orphan.Name)
+	}
+
+	dataEngineTypeStr, ok := orphan.Spec.Parameters[longhorn.OrphanDataEngineType]
+	if !ok {
+		return "", "", fmt.Errorf("failed to get data engine type for runtime instance orphan %v", orphan.Name)
+	}
+	switch dataEngineType = longhorn.DataEngineType(dataEngineTypeStr); dataEngineType {
+	case longhorn.DataEngineTypeV1, longhorn.DataEngineTypeV2:
+		// supported data engine type
 	default:
-		err = fmt.Errorf("unknown orphan type %v", orphan.Spec.Type)
+		return "", "", fmt.Errorf("unknown data engine type %v for runtime instance orphan %v", dataEngineTypeStr, orphan.Name)
 	}
 
-	if err == nil || (err != nil && apierrors.IsNotFound(err)) {
-		return nil
+	return name, dataEngineType, nil
+}
+
+func (oc *OrphanController) deleteRuntimeInstance(imc *engineapi.InstanceManagerClient, instanceName string, instanceKind longhorn.InstanceManagerType, engineType longhorn.DataEngineType) (err error) {
+	// There is a delay between deletion initiation and state/InstanceManager update,
+	// this function may be called multiple times before given instance exits.
+
+	oc.logger.Infof("Deleting runtime instance %v", instanceName)
+
+	defer func() {
+		if err != nil {
+			err = errors.Wrapf(err, "cannot delete runtime instance %v", instanceName)
+		}
+	}()
+
+	err = imc.InstanceDelete(engineType, instanceName, string(instanceKind), "", false)
+	if err != nil && !types.ErrorIsNotFound(err) {
+		return err
 	}
 
-	return err
+	return nil
+}
+
+func (oc *OrphanController) confirmOrphanRuntimeInstanceCleanup(imc *engineapi.InstanceManagerClient, instanceName string, imType longhorn.InstanceManagerType, engineType longhorn.DataEngineType) (isCleanupComplete bool, err error) {
+	defer func() {
+		if err != nil {
+			err = errors.Wrapf(err, "cannot confirm cleanup result of runtime instance %v", instanceName)
+		}
+	}()
+
+	_, err = imc.InstanceGet(engineType, instanceName, string(imType))
+	isCleanupComplete = false
+	if err != nil && types.ErrorIsNotFound(err) {
+		isCleanupComplete = true
+		err = nil
+	}
+	return isCleanupComplete, err
path: controller/orphan_controller.go
user: login: c3y1huang
body: NIT: consider tweaking this for better readabiliy and maintainablity.

```suggestion
	_, err = imc.InstanceGet(engineType, instanceName, string(imType))
	switch {
	case err == nil:
		return false, nil // instance still exists - cleanup not complete.
	case types.ErrorIsNotFound(err):
		return true, nil // instance not found - cleanup completed.
	default:
		return false, err // Unexpected error - cleanup status unknown.
	}
```
created_at: 2025-04-18T02:18:26Z
author_association: CONTRIBUTOR
diff_hunk: @@ -360,8 +535,13 @@ func (oc *OrphanController) DeleteV2ReplicaInstance(diskName, diskUUID, diskDriv
 }
 
 func (oc *OrphanController) updateConditions(orphan *longhorn.Orphan) error {
-	if err := oc.updateDataCleanableCondition(orphan); err != nil {
-		return err
+	switch orphan.Spec.Type {
path: controller/orphan_controller.go
user: login: c3y1huang
body: Consider handling the default case and logging unexpected type.
created_at: 2025-04-18T02:21:06Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1052,14 +1055,30 @@ var (
 	SettingDefinitionOrphanAutoDeletion = SettingDefinition{
 		DisplayName: "Orphan Auto-Deletion",
 		Description: "This setting allows Longhorn to delete the orphan resource and its corresponding orphaned data automatically. \n\n" +
-			"Orphan resources on down or unknown nodes will not be cleaned up automatically. \n\n",
+			"Orphan resources on down or unknown nodes will not be cleaned up automatically. \n\n" +
+			fmt.Sprintf("Deprecated: enable \"%s\" in %s instead. \n\n", OrphanResourceTypeReplicaData, SettingNameOrphanResourceAutoDeletion),
 		Category: SettingCategoryOrphan,
 		Type:     SettingTypeBool,
 		Required: true,
-		ReadOnly: false,
+		ReadOnly: true,
 		Default:  "false",
 	}
 
+	SettingDefinitionOrphanResourceAutoDeletion = SettingDefinition{
+		DisplayName: "Orphan Resource Auto-Deletion",
+		Description: "This setting allows Longhorn to delete the orphan resource and its corresponding orphaned resource automatically. \n\n" +
+			"Orphan resources on down or unknown nodes will not be cleaned up automatically. \n\n" +
+			"List the enabled resource types in a semicolon-separate list. Available items are: \n\n" +
+			"  - **replicaData**: replica data store \n\n" +
+			"  - **engineInstance**: engine runtime instance \n\n" +
+			"  - **replicaInstance**: replica runtime instance \n\n",
path: types/setting.go
user: login: c3y1huang
body: This is user-facing content. Consider aligning with other setting descriptions for consistency.
```suggestion
			"List the enabled resource types in a semicolon-separate list. \n\n" +
			"Available items are: \n\n" +
			"- **replicaData**: replica data store \n\n" +
			"- **engineInstance**: engine runtime instance \n\n" +
			"- **replicaInstance**: replica runtime instance \n\n",
```
created_at: 2025-04-18T02:33:33Z
author_association: CONTRIBUTOR
diff_hunk: @@ -269,48 +303,187 @@ func (oc *OrphanController) isResponsibleFor(orphan *longhorn.Orphan) bool {
 	return isControllerResponsibleFor(oc.controllerID, oc.ds, orphan.Name, orphan.Spec.NodeID, orphan.Status.OwnerID)
 }
 
-func (oc *OrphanController) cleanupOrphanedData(orphan *longhorn.Orphan) (err error) {
+func (oc *OrphanController) cleanupOrphanedResource(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
 	log := getLoggerForOrphan(oc.logger, orphan)
 
 	defer func() {
 		if err == nil {
 			return
 		}
 
-		err = errors.Wrapf(err, "failed to delete orphan %v data", orphan.Name)
+		err = errors.Wrapf(err, "failed to delete %v orphan %v", orphan.Spec.Type, orphan.Name)
 		orphan.Status.Conditions = types.SetCondition(orphan.Status.Conditions,
 			longhorn.OrphanConditionTypeError, longhorn.ConditionStatusTrue, "", err.Error())
 	}()
 
 	// Make sure if the orphan nodeID and controller ID are same.
 	// If NO, just delete the orphan resource object and don't touch the data.
 	if orphan.Spec.NodeID != oc.controllerID {
-		log.Infof("Orphan nodeID %v is different from controllerID %v, so just delete the orphan resource object",
-			orphan.Name, oc.controllerID)
-		return nil
+		log.WithField("orphanType", orphan.Spec.Type).Infof("Orphan nodeID %v is on %v. Delete the orphan resource object.",
+			orphan.Name, orphan.Spec.NodeID)
+		return true, nil
 	}
 
-	if types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeDataCleanable).Status !=
-		longhorn.ConditionStatusTrue {
-		log.Infof("Only delete orphan %v resource object and do not delete the orphaned data", orphan.Name)
-		return nil
+	switch orphan.Spec.Type {
+	case longhorn.OrphanTypeEngineInstance:
+		isCleanupComplete, err = oc.cleanupOrphanedEngineInstance(orphan)
+	case longhorn.OrphanTypeReplicaInstance:
+		isCleanupComplete, err = oc.cleanupOrphanedReplicaInstance(orphan)
+	case longhorn.OrphanTypeReplicaDataStore:
+		if types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeDataCleanable).Status !=
+			longhorn.ConditionStatusTrue {
+			log.Infof("Only delete orphan %v resource object and do not delete the orphaned data store", orphan.Name)
+			return true, nil
+		}
+		err = oc.deleteOrphanedReplicaDataStore(orphan)
+		if err == nil || (err != nil && apierrors.IsNotFound(err)) {
+			isCleanupComplete = true
+			err = nil
+		}
+	default:
+		isCleanupComplete = false
+		err = fmt.Errorf("unknown orphan type %v of orphan %v", orphan.Spec.Type, orphan.Name)
 	}
 
-	switch orphan.Spec.Type {
-	case longhorn.OrphanTypeReplica:
-		err = oc.deleteOrphanedReplica(orphan)
+	return isCleanupComplete, err
+}
+
+func (oc *OrphanController) cleanupOrphanedEngineInstance(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
+	defer func() {
+		if err != nil {
+			err = errors.Wrapf(err, "cannot cleanup engine instance orphan %v", orphan.Name)
+		}
+	}()
+
+	instance, deType, err := oc.extractOrphanedRuntimeInfo(orphan)
+	if err != nil {
+		return false, err
+	}
+
+	var spec *longhorn.InstanceSpec
+	if engineCR, err := oc.ds.GetEngineRO(instance); err != nil {
+		if !apierrors.IsNotFound(err) {
+			return false, err
+		}
+		spec = nil
+	} else {
+		spec = &engineCR.Spec.InstanceSpec
+	}
+	return oc.cleanupOrphanedRuntimeInstance(orphan, instance, longhorn.InstanceManagerTypeEngine, deType, spec)
+}
+
+func (oc *OrphanController) cleanupOrphanedReplicaInstance(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
+	defer func() {
+		if err != nil {
+			err = errors.Wrapf(err, "cannot cleanup replica instance orphan %v", orphan.Name)
+		}
+	}()
+
+	instance, deType, err := oc.extractOrphanedRuntimeInfo(orphan)
+	if err != nil {
+		return false, err
+	}
+
+	var spec *longhorn.InstanceSpec
+	if replicaCR, err := oc.ds.GetReplicaRO(instance); err != nil {
+		if !apierrors.IsNotFound(err) {
+			return false, err
+		}
+		spec = nil
+	} else {
+		spec = &replicaCR.Spec.InstanceSpec
+	}
+	return oc.cleanupOrphanedRuntimeInstance(orphan, instance, longhorn.InstanceManagerTypeReplica, deType, spec)
+}
+
+func (oc *OrphanController) cleanupOrphanedRuntimeInstance(orphan *longhorn.Orphan, instance string, imType longhorn.InstanceManagerType, deType longhorn.DataEngineType, instanceCRSpec *longhorn.InstanceSpec) (isCleanupComplete bool, err error) {
+	if instanceCRSpec != nil && instanceCRSpec.NodeID == oc.controllerID {
+		// Instance is scheduled back to current node and is no longer considered orphaned.
+		// Finalize the orphan CR directly without terminating the instance process.
+		return true, nil
+	}
+
+	im, err := oc.ds.GetRunningInstanceManagerByNodeRO(oc.controllerID, deType)
+	if err != nil {
+		oc.logger.WithError(err).Infof("No running instance manager for node %v for deleting runtime instance orphan %v", oc.controllerID, orphan.Name)
+		return true, nil
+	}
+	imc, err := engineapi.NewInstanceManagerClient(im, false)
+	if err != nil {
+		return false, err
+	}
+	defer func() {
+		if closeErr := imc.Close(); closeErr != nil {
+			oc.logger.WithError(closeErr).Error("failed to close instance manager client")
+		}
+	}()
+
+	runtimeInstanceExist := types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeRuntimeInstanceState).Status == longhorn.ConditionStatusTrue
+	if runtimeInstanceExist {
+		if err = oc.deleteRuntimeInstance(imc, instance, imType, deType); err != nil {
+			return false, err
+		}
+	}
+	return oc.confirmOrphanRuntimeInstanceCleanup(imc, instance, imType, deType)
+}
+
+func (oc *OrphanController) extractOrphanedRuntimeInfo(orphan *longhorn.Orphan) (name string, dataEngineType longhorn.DataEngineType, err error) {
+	name, ok := orphan.Spec.Parameters[longhorn.OrphanInstanceName]
+	if !ok {
+		return "", "", fmt.Errorf("failed to get instance name for runtime instance orphan %v", orphan.Name)
+	}
+
+	dataEngineTypeStr, ok := orphan.Spec.Parameters[longhorn.OrphanDataEngineType]
+	if !ok {
+		return "", "", fmt.Errorf("failed to get data engine type for runtime instance orphan %v", orphan.Name)
+	}
+	switch dataEngineType = longhorn.DataEngineType(dataEngineTypeStr); dataEngineType {
+	case longhorn.DataEngineTypeV1, longhorn.DataEngineTypeV2:
+		// supported data engine type
 	default:
-		err = fmt.Errorf("unknown orphan type %v", orphan.Spec.Type)
+		return "", "", fmt.Errorf("unknown data engine type %v for runtime instance orphan %v", dataEngineTypeStr, orphan.Name)
path: controller/orphan_controller.go
user: login: COLDTURNIP
body: Good idea, will add this.
created_at: 2025-04-18T03:01:20Z
author_association: MEMBER
diff_hunk: @@ -269,48 +303,187 @@ func (oc *OrphanController) isResponsibleFor(orphan *longhorn.Orphan) bool {
 	return isControllerResponsibleFor(oc.controllerID, oc.ds, orphan.Name, orphan.Spec.NodeID, orphan.Status.OwnerID)
 }
 
-func (oc *OrphanController) cleanupOrphanedData(orphan *longhorn.Orphan) (err error) {
+func (oc *OrphanController) cleanupOrphanedResource(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
 	log := getLoggerForOrphan(oc.logger, orphan)
 
 	defer func() {
 		if err == nil {
 			return
 		}
 
-		err = errors.Wrapf(err, "failed to delete orphan %v data", orphan.Name)
+		err = errors.Wrapf(err, "failed to delete %v orphan %v", orphan.Spec.Type, orphan.Name)
 		orphan.Status.Conditions = types.SetCondition(orphan.Status.Conditions,
 			longhorn.OrphanConditionTypeError, longhorn.ConditionStatusTrue, "", err.Error())
 	}()
 
 	// Make sure if the orphan nodeID and controller ID are same.
 	// If NO, just delete the orphan resource object and don't touch the data.
 	if orphan.Spec.NodeID != oc.controllerID {
-		log.Infof("Orphan nodeID %v is different from controllerID %v, so just delete the orphan resource object",
-			orphan.Name, oc.controllerID)
-		return nil
+		log.WithField("orphanType", orphan.Spec.Type).Infof("Orphan nodeID %v is on %v. Delete the orphan resource object.",
+			orphan.Name, orphan.Spec.NodeID)
+		return true, nil
 	}
 
-	if types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeDataCleanable).Status !=
-		longhorn.ConditionStatusTrue {
-		log.Infof("Only delete orphan %v resource object and do not delete the orphaned data", orphan.Name)
-		return nil
+	switch orphan.Spec.Type {
+	case longhorn.OrphanTypeEngineInstance:
+		isCleanupComplete, err = oc.cleanupOrphanedEngineInstance(orphan)
+	case longhorn.OrphanTypeReplicaInstance:
+		isCleanupComplete, err = oc.cleanupOrphanedReplicaInstance(orphan)
+	case longhorn.OrphanTypeReplicaDataStore:
+		if types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeDataCleanable).Status !=
+			longhorn.ConditionStatusTrue {
+			log.Infof("Only delete orphan %v resource object and do not delete the orphaned data store", orphan.Name)
+			return true, nil
+		}
+		err = oc.deleteOrphanedReplicaDataStore(orphan)
+		if err == nil || (err != nil && apierrors.IsNotFound(err)) {
+			isCleanupComplete = true
+			err = nil
+		}
+	default:
+		isCleanupComplete = false
+		err = fmt.Errorf("unknown orphan type %v of orphan %v", orphan.Spec.Type, orphan.Name)
 	}
 
-	switch orphan.Spec.Type {
-	case longhorn.OrphanTypeReplica:
-		err = oc.deleteOrphanedReplica(orphan)
+	return isCleanupComplete, err
+}
+
+func (oc *OrphanController) cleanupOrphanedEngineInstance(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
+	defer func() {
+		if err != nil {
+			err = errors.Wrapf(err, "cannot cleanup engine instance orphan %v", orphan.Name)
+		}
+	}()
+
+	instance, deType, err := oc.extractOrphanedRuntimeInfo(orphan)
+	if err != nil {
+		return false, err
+	}
+
+	var spec *longhorn.InstanceSpec
+	if engineCR, err := oc.ds.GetEngineRO(instance); err != nil {
+		if !apierrors.IsNotFound(err) {
+			return false, err
+		}
+		spec = nil
+	} else {
+		spec = &engineCR.Spec.InstanceSpec
+	}
+	return oc.cleanupOrphanedRuntimeInstance(orphan, instance, longhorn.InstanceManagerTypeEngine, deType, spec)
+}
+
+func (oc *OrphanController) cleanupOrphanedReplicaInstance(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
+	defer func() {
+		if err != nil {
+			err = errors.Wrapf(err, "cannot cleanup replica instance orphan %v", orphan.Name)
+		}
+	}()
+
+	instance, deType, err := oc.extractOrphanedRuntimeInfo(orphan)
+	if err != nil {
+		return false, err
+	}
+
+	var spec *longhorn.InstanceSpec
+	if replicaCR, err := oc.ds.GetReplicaRO(instance); err != nil {
+		if !apierrors.IsNotFound(err) {
+			return false, err
+		}
+		spec = nil
+	} else {
+		spec = &replicaCR.Spec.InstanceSpec
+	}
+	return oc.cleanupOrphanedRuntimeInstance(orphan, instance, longhorn.InstanceManagerTypeReplica, deType, spec)
+}
+
+func (oc *OrphanController) cleanupOrphanedRuntimeInstance(orphan *longhorn.Orphan, instance string, imType longhorn.InstanceManagerType, deType longhorn.DataEngineType, instanceCRSpec *longhorn.InstanceSpec) (isCleanupComplete bool, err error) {
+	if instanceCRSpec != nil && instanceCRSpec.NodeID == oc.controllerID {
+		// Instance is scheduled back to current node and is no longer considered orphaned.
+		// Finalize the orphan CR directly without terminating the instance process.
+		return true, nil
+	}
+
+	im, err := oc.ds.GetRunningInstanceManagerByNodeRO(oc.controllerID, deType)
+	if err != nil {
+		oc.logger.WithError(err).Infof("No running instance manager for node %v for deleting runtime instance orphan %v", oc.controllerID, orphan.Name)
+		return true, nil
+	}
+	imc, err := engineapi.NewInstanceManagerClient(im, false)
+	if err != nil {
+		return false, err
+	}
+	defer func() {
+		if closeErr := imc.Close(); closeErr != nil {
+			oc.logger.WithError(closeErr).Error("failed to close instance manager client")
+		}
+	}()
+
+	runtimeInstanceExist := types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeRuntimeInstanceState).Status == longhorn.ConditionStatusTrue
+	if runtimeInstanceExist {
+		if err = oc.deleteRuntimeInstance(imc, instance, imType, deType); err != nil {
+			return false, err
+		}
+	}
+	return oc.confirmOrphanRuntimeInstanceCleanup(imc, instance, imType, deType)
+}
+
+func (oc *OrphanController) extractOrphanedRuntimeInfo(orphan *longhorn.Orphan) (name string, dataEngineType longhorn.DataEngineType, err error) {
+	name, ok := orphan.Spec.Parameters[longhorn.OrphanInstanceName]
+	if !ok {
+		return "", "", fmt.Errorf("failed to get instance name for runtime instance orphan %v", orphan.Name)
+	}
+
+	dataEngineTypeStr, ok := orphan.Spec.Parameters[longhorn.OrphanDataEngineType]
+	if !ok {
+		return "", "", fmt.Errorf("failed to get data engine type for runtime instance orphan %v", orphan.Name)
+	}
+	switch dataEngineType = longhorn.DataEngineType(dataEngineTypeStr); dataEngineType {
+	case longhorn.DataEngineTypeV1, longhorn.DataEngineTypeV2:
+		// supported data engine type
 	default:
-		err = fmt.Errorf("unknown orphan type %v", orphan.Spec.Type)
+		return "", "", fmt.Errorf("unknown data engine type %v for runtime instance orphan %v", dataEngineTypeStr, orphan.Name)
path: controller/orphan_controller.go
user: login: COLDTURNIP
body: We already have orphan type check in validator. Still check here to prevent potential programming mistake.
created_at: 2025-04-18T03:16:30Z
author_association: MEMBER
diff_hunk: @@ -269,48 +303,187 @@ func (oc *OrphanController) isResponsibleFor(orphan *longhorn.Orphan) bool {
 	return isControllerResponsibleFor(oc.controllerID, oc.ds, orphan.Name, orphan.Spec.NodeID, orphan.Status.OwnerID)
 }
 
-func (oc *OrphanController) cleanupOrphanedData(orphan *longhorn.Orphan) (err error) {
+func (oc *OrphanController) cleanupOrphanedResource(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
 	log := getLoggerForOrphan(oc.logger, orphan)
 
 	defer func() {
 		if err == nil {
 			return
 		}
 
-		err = errors.Wrapf(err, "failed to delete orphan %v data", orphan.Name)
+		err = errors.Wrapf(err, "failed to delete %v orphan %v", orphan.Spec.Type, orphan.Name)
 		orphan.Status.Conditions = types.SetCondition(orphan.Status.Conditions,
 			longhorn.OrphanConditionTypeError, longhorn.ConditionStatusTrue, "", err.Error())
 	}()
 
 	// Make sure if the orphan nodeID and controller ID are same.
 	// If NO, just delete the orphan resource object and don't touch the data.
 	if orphan.Spec.NodeID != oc.controllerID {
-		log.Infof("Orphan nodeID %v is different from controllerID %v, so just delete the orphan resource object",
-			orphan.Name, oc.controllerID)
-		return nil
+		log.WithField("orphanType", orphan.Spec.Type).Infof("Orphan nodeID %v is on %v. Delete the orphan resource object.",
+			orphan.Name, orphan.Spec.NodeID)
+		return true, nil
 	}
 
-	if types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeDataCleanable).Status !=
-		longhorn.ConditionStatusTrue {
-		log.Infof("Only delete orphan %v resource object and do not delete the orphaned data", orphan.Name)
-		return nil
+	switch orphan.Spec.Type {
+	case longhorn.OrphanTypeEngineInstance:
+		isCleanupComplete, err = oc.cleanupOrphanedEngineInstance(orphan)
+	case longhorn.OrphanTypeReplicaInstance:
+		isCleanupComplete, err = oc.cleanupOrphanedReplicaInstance(orphan)
+	case longhorn.OrphanTypeReplicaDataStore:
+		if types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeDataCleanable).Status !=
+			longhorn.ConditionStatusTrue {
+			log.Infof("Only delete orphan %v resource object and do not delete the orphaned data store", orphan.Name)
+			return true, nil
+		}
+		err = oc.deleteOrphanedReplicaDataStore(orphan)
+		if err == nil || (err != nil && apierrors.IsNotFound(err)) {
+			isCleanupComplete = true
+			err = nil
+		}
+	default:
+		isCleanupComplete = false
+		err = fmt.Errorf("unknown orphan type %v of orphan %v", orphan.Spec.Type, orphan.Name)
 	}
 
-	switch orphan.Spec.Type {
-	case longhorn.OrphanTypeReplica:
-		err = oc.deleteOrphanedReplica(orphan)
+	return isCleanupComplete, err
+}
+
+func (oc *OrphanController) cleanupOrphanedEngineInstance(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
+	defer func() {
+		if err != nil {
+			err = errors.Wrapf(err, "cannot cleanup engine instance orphan %v", orphan.Name)
+		}
+	}()
+
+	instance, deType, err := oc.extractOrphanedRuntimeInfo(orphan)
+	if err != nil {
+		return false, err
+	}
+
+	var spec *longhorn.InstanceSpec
+	if engineCR, err := oc.ds.GetEngineRO(instance); err != nil {
+		if !apierrors.IsNotFound(err) {
+			return false, err
+		}
+		spec = nil
+	} else {
+		spec = &engineCR.Spec.InstanceSpec
+	}
+	return oc.cleanupOrphanedRuntimeInstance(orphan, instance, longhorn.InstanceManagerTypeEngine, deType, spec)
+}
+
+func (oc *OrphanController) cleanupOrphanedReplicaInstance(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
+	defer func() {
+		if err != nil {
+			err = errors.Wrapf(err, "cannot cleanup replica instance orphan %v", orphan.Name)
+		}
+	}()
+
+	instance, deType, err := oc.extractOrphanedRuntimeInfo(orphan)
+	if err != nil {
+		return false, err
+	}
+
+	var spec *longhorn.InstanceSpec
+	if replicaCR, err := oc.ds.GetReplicaRO(instance); err != nil {
+		if !apierrors.IsNotFound(err) {
+			return false, err
+		}
+		spec = nil
+	} else {
+		spec = &replicaCR.Spec.InstanceSpec
+	}
+	return oc.cleanupOrphanedRuntimeInstance(orphan, instance, longhorn.InstanceManagerTypeReplica, deType, spec)
+}
+
+func (oc *OrphanController) cleanupOrphanedRuntimeInstance(orphan *longhorn.Orphan, instance string, imType longhorn.InstanceManagerType, deType longhorn.DataEngineType, instanceCRSpec *longhorn.InstanceSpec) (isCleanupComplete bool, err error) {
+	if instanceCRSpec != nil && instanceCRSpec.NodeID == oc.controllerID {
+		// Instance is scheduled back to current node and is no longer considered orphaned.
+		// Finalize the orphan CR directly without terminating the instance process.
+		return true, nil
+	}
+
+	im, err := oc.ds.GetRunningInstanceManagerByNodeRO(oc.controllerID, deType)
+	if err != nil {
+		oc.logger.WithError(err).Infof("No running instance manager for node %v for deleting runtime instance orphan %v", oc.controllerID, orphan.Name)
+		return true, nil
+	}
+	imc, err := engineapi.NewInstanceManagerClient(im, false)
+	if err != nil {
+		return false, err
+	}
+	defer func() {
+		if closeErr := imc.Close(); closeErr != nil {
+			oc.logger.WithError(closeErr).Error("failed to close instance manager client")
+		}
+	}()
+
+	runtimeInstanceExist := types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeRuntimeInstanceState).Status == longhorn.ConditionStatusTrue
+	if runtimeInstanceExist {
+		if err = oc.deleteRuntimeInstance(imc, instance, imType, deType); err != nil {
+			return false, err
+		}
+	}
+	return oc.confirmOrphanRuntimeInstanceCleanup(imc, instance, imType, deType)
+}
+
+func (oc *OrphanController) extractOrphanedRuntimeInfo(orphan *longhorn.Orphan) (name string, dataEngineType longhorn.DataEngineType, err error) {
+	name, ok := orphan.Spec.Parameters[longhorn.OrphanInstanceName]
+	if !ok {
+		return "", "", fmt.Errorf("failed to get instance name for runtime instance orphan %v", orphan.Name)
+	}
+
+	dataEngineTypeStr, ok := orphan.Spec.Parameters[longhorn.OrphanDataEngineType]
+	if !ok {
+		return "", "", fmt.Errorf("failed to get data engine type for runtime instance orphan %v", orphan.Name)
+	}
+	switch dataEngineType = longhorn.DataEngineType(dataEngineTypeStr); dataEngineType {
+	case longhorn.DataEngineTypeV1, longhorn.DataEngineTypeV2:
+		// supported data engine type
 	default:
-		err = fmt.Errorf("unknown orphan type %v", orphan.Spec.Type)
+		return "", "", fmt.Errorf("unknown data engine type %v for runtime instance orphan %v", dataEngineTypeStr, orphan.Name)
path: controller/orphan_controller.go
user: login: c3y1huang
body: Could you point me to where the validation of `dataEngineType` happens in the webhook?

Also, if it is already being validated in the webhook, could you clarify the scenario about `potential programming mistake`? I just want to make sure we are not over-engineering this.
created_at: 2025-04-18T04:36:32Z
author_association: CONTRIBUTOR
diff_hunk: @@ -269,48 +303,187 @@ func (oc *OrphanController) isResponsibleFor(orphan *longhorn.Orphan) bool {
 	return isControllerResponsibleFor(oc.controllerID, oc.ds, orphan.Name, orphan.Spec.NodeID, orphan.Status.OwnerID)
 }
 
-func (oc *OrphanController) cleanupOrphanedData(orphan *longhorn.Orphan) (err error) {
+func (oc *OrphanController) cleanupOrphanedResource(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
 	log := getLoggerForOrphan(oc.logger, orphan)
 
 	defer func() {
 		if err == nil {
 			return
 		}
 
-		err = errors.Wrapf(err, "failed to delete orphan %v data", orphan.Name)
+		err = errors.Wrapf(err, "failed to delete %v orphan %v", orphan.Spec.Type, orphan.Name)
 		orphan.Status.Conditions = types.SetCondition(orphan.Status.Conditions,
 			longhorn.OrphanConditionTypeError, longhorn.ConditionStatusTrue, "", err.Error())
 	}()
 
 	// Make sure if the orphan nodeID and controller ID are same.
 	// If NO, just delete the orphan resource object and don't touch the data.
 	if orphan.Spec.NodeID != oc.controllerID {
-		log.Infof("Orphan nodeID %v is different from controllerID %v, so just delete the orphan resource object",
-			orphan.Name, oc.controllerID)
-		return nil
+		log.WithField("orphanType", orphan.Spec.Type).Infof("Orphan nodeID %v is on %v. Delete the orphan resource object.",
+			orphan.Name, orphan.Spec.NodeID)
+		return true, nil
 	}
 
-	if types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeDataCleanable).Status !=
-		longhorn.ConditionStatusTrue {
-		log.Infof("Only delete orphan %v resource object and do not delete the orphaned data", orphan.Name)
-		return nil
+	switch orphan.Spec.Type {
+	case longhorn.OrphanTypeEngineInstance:
+		isCleanupComplete, err = oc.cleanupOrphanedEngineInstance(orphan)
+	case longhorn.OrphanTypeReplicaInstance:
+		isCleanupComplete, err = oc.cleanupOrphanedReplicaInstance(orphan)
+	case longhorn.OrphanTypeReplicaDataStore:
+		if types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeDataCleanable).Status !=
+			longhorn.ConditionStatusTrue {
+			log.Infof("Only delete orphan %v resource object and do not delete the orphaned data store", orphan.Name)
+			return true, nil
+		}
+		err = oc.deleteOrphanedReplicaDataStore(orphan)
+		if err == nil || (err != nil && apierrors.IsNotFound(err)) {
+			isCleanupComplete = true
+			err = nil
+		}
+	default:
+		isCleanupComplete = false
+		err = fmt.Errorf("unknown orphan type %v of orphan %v", orphan.Spec.Type, orphan.Name)
 	}
 
-	switch orphan.Spec.Type {
-	case longhorn.OrphanTypeReplica:
-		err = oc.deleteOrphanedReplica(orphan)
+	return isCleanupComplete, err
+}
+
+func (oc *OrphanController) cleanupOrphanedEngineInstance(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
+	defer func() {
+		if err != nil {
+			err = errors.Wrapf(err, "cannot cleanup engine instance orphan %v", orphan.Name)
+		}
+	}()
+
+	instance, deType, err := oc.extractOrphanedRuntimeInfo(orphan)
+	if err != nil {
+		return false, err
+	}
+
+	var spec *longhorn.InstanceSpec
+	if engineCR, err := oc.ds.GetEngineRO(instance); err != nil {
+		if !apierrors.IsNotFound(err) {
+			return false, err
+		}
+		spec = nil
+	} else {
+		spec = &engineCR.Spec.InstanceSpec
+	}
+	return oc.cleanupOrphanedRuntimeInstance(orphan, instance, longhorn.InstanceManagerTypeEngine, deType, spec)
+}
+
+func (oc *OrphanController) cleanupOrphanedReplicaInstance(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
+	defer func() {
+		if err != nil {
+			err = errors.Wrapf(err, "cannot cleanup replica instance orphan %v", orphan.Name)
+		}
+	}()
+
+	instance, deType, err := oc.extractOrphanedRuntimeInfo(orphan)
+	if err != nil {
+		return false, err
+	}
+
+	var spec *longhorn.InstanceSpec
+	if replicaCR, err := oc.ds.GetReplicaRO(instance); err != nil {
+		if !apierrors.IsNotFound(err) {
+			return false, err
+		}
+		spec = nil
+	} else {
+		spec = &replicaCR.Spec.InstanceSpec
+	}
+	return oc.cleanupOrphanedRuntimeInstance(orphan, instance, longhorn.InstanceManagerTypeReplica, deType, spec)
+}
+
+func (oc *OrphanController) cleanupOrphanedRuntimeInstance(orphan *longhorn.Orphan, instance string, imType longhorn.InstanceManagerType, deType longhorn.DataEngineType, instanceCRSpec *longhorn.InstanceSpec) (isCleanupComplete bool, err error) {
+	if instanceCRSpec != nil && instanceCRSpec.NodeID == oc.controllerID {
+		// Instance is scheduled back to current node and is no longer considered orphaned.
+		// Finalize the orphan CR directly without terminating the instance process.
+		return true, nil
+	}
+
+	im, err := oc.ds.GetRunningInstanceManagerByNodeRO(oc.controllerID, deType)
+	if err != nil {
+		oc.logger.WithError(err).Infof("No running instance manager for node %v for deleting runtime instance orphan %v", oc.controllerID, orphan.Name)
+		return true, nil
+	}
+	imc, err := engineapi.NewInstanceManagerClient(im, false)
+	if err != nil {
+		return false, err
+	}
+	defer func() {
+		if closeErr := imc.Close(); closeErr != nil {
+			oc.logger.WithError(closeErr).Error("failed to close instance manager client")
+		}
+	}()
+
+	runtimeInstanceExist := types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeRuntimeInstanceState).Status == longhorn.ConditionStatusTrue
+	if runtimeInstanceExist {
+		if err = oc.deleteRuntimeInstance(imc, instance, imType, deType); err != nil {
+			return false, err
+		}
+	}
+	return oc.confirmOrphanRuntimeInstanceCleanup(imc, instance, imType, deType)
+}
+
+func (oc *OrphanController) extractOrphanedRuntimeInfo(orphan *longhorn.Orphan) (name string, dataEngineType longhorn.DataEngineType, err error) {
+	name, ok := orphan.Spec.Parameters[longhorn.OrphanInstanceName]
+	if !ok {
+		return "", "", fmt.Errorf("failed to get instance name for runtime instance orphan %v", orphan.Name)
+	}
+
+	dataEngineTypeStr, ok := orphan.Spec.Parameters[longhorn.OrphanDataEngineType]
+	if !ok {
+		return "", "", fmt.Errorf("failed to get data engine type for runtime instance orphan %v", orphan.Name)
+	}
+	switch dataEngineType = longhorn.DataEngineType(dataEngineTypeStr); dataEngineType {
+	case longhorn.DataEngineTypeV1, longhorn.DataEngineTypeV2:
+		// supported data engine type
 	default:
-		err = fmt.Errorf("unknown orphan type %v", orphan.Spec.Type)
+		return "", "", fmt.Errorf("unknown data engine type %v for runtime instance orphan %v", dataEngineTypeStr, orphan.Name)
path: controller/orphan_controller.go
user: login: COLDTURNIP
body: Sorry for misleading. We have checked orphan type in webhook, but not the data engine type.

Though the type is checked in the webhook, printing error log for unknown type here would prevent developers forgot check this in the webhook in the future. At least we'll get errors, instead of leaving it silently.
created_at: 2025-04-18T08:41:38Z
author_association: MEMBER
diff_hunk: @@ -269,48 +303,187 @@ func (oc *OrphanController) isResponsibleFor(orphan *longhorn.Orphan) bool {
 	return isControllerResponsibleFor(oc.controllerID, oc.ds, orphan.Name, orphan.Spec.NodeID, orphan.Status.OwnerID)
 }
 
-func (oc *OrphanController) cleanupOrphanedData(orphan *longhorn.Orphan) (err error) {
+func (oc *OrphanController) cleanupOrphanedResource(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
 	log := getLoggerForOrphan(oc.logger, orphan)
 
 	defer func() {
 		if err == nil {
 			return
 		}
 
-		err = errors.Wrapf(err, "failed to delete orphan %v data", orphan.Name)
+		err = errors.Wrapf(err, "failed to delete %v orphan %v", orphan.Spec.Type, orphan.Name)
 		orphan.Status.Conditions = types.SetCondition(orphan.Status.Conditions,
 			longhorn.OrphanConditionTypeError, longhorn.ConditionStatusTrue, "", err.Error())
 	}()
 
 	// Make sure if the orphan nodeID and controller ID are same.
 	// If NO, just delete the orphan resource object and don't touch the data.
 	if orphan.Spec.NodeID != oc.controllerID {
-		log.Infof("Orphan nodeID %v is different from controllerID %v, so just delete the orphan resource object",
-			orphan.Name, oc.controllerID)
-		return nil
+		log.WithField("orphanType", orphan.Spec.Type).Infof("Orphan nodeID %v is on %v. Delete the orphan resource object.",
+			orphan.Name, orphan.Spec.NodeID)
+		return true, nil
 	}
 
-	if types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeDataCleanable).Status !=
-		longhorn.ConditionStatusTrue {
-		log.Infof("Only delete orphan %v resource object and do not delete the orphaned data", orphan.Name)
-		return nil
+	switch orphan.Spec.Type {
+	case longhorn.OrphanTypeEngineInstance:
+		isCleanupComplete, err = oc.cleanupOrphanedEngineInstance(orphan)
+	case longhorn.OrphanTypeReplicaInstance:
+		isCleanupComplete, err = oc.cleanupOrphanedReplicaInstance(orphan)
+	case longhorn.OrphanTypeReplicaDataStore:
+		if types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeDataCleanable).Status !=
+			longhorn.ConditionStatusTrue {
+			log.Infof("Only delete orphan %v resource object and do not delete the orphaned data store", orphan.Name)
+			return true, nil
+		}
+		err = oc.deleteOrphanedReplicaDataStore(orphan)
+		if err == nil || (err != nil && apierrors.IsNotFound(err)) {
+			isCleanupComplete = true
+			err = nil
+		}
+	default:
+		isCleanupComplete = false
+		err = fmt.Errorf("unknown orphan type %v of orphan %v", orphan.Spec.Type, orphan.Name)
 	}
 
-	switch orphan.Spec.Type {
-	case longhorn.OrphanTypeReplica:
-		err = oc.deleteOrphanedReplica(orphan)
+	return isCleanupComplete, err
+}
+
+func (oc *OrphanController) cleanupOrphanedEngineInstance(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
+	defer func() {
+		if err != nil {
+			err = errors.Wrapf(err, "cannot cleanup engine instance orphan %v", orphan.Name)
+		}
+	}()
+
+	instance, deType, err := oc.extractOrphanedRuntimeInfo(orphan)
+	if err != nil {
+		return false, err
+	}
+
+	var spec *longhorn.InstanceSpec
+	if engineCR, err := oc.ds.GetEngineRO(instance); err != nil {
+		if !apierrors.IsNotFound(err) {
+			return false, err
+		}
+		spec = nil
+	} else {
+		spec = &engineCR.Spec.InstanceSpec
+	}
+	return oc.cleanupOrphanedRuntimeInstance(orphan, instance, longhorn.InstanceManagerTypeEngine, deType, spec)
+}
+
+func (oc *OrphanController) cleanupOrphanedReplicaInstance(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
+	defer func() {
+		if err != nil {
+			err = errors.Wrapf(err, "cannot cleanup replica instance orphan %v", orphan.Name)
+		}
+	}()
+
+	instance, deType, err := oc.extractOrphanedRuntimeInfo(orphan)
+	if err != nil {
+		return false, err
+	}
+
+	var spec *longhorn.InstanceSpec
+	if replicaCR, err := oc.ds.GetReplicaRO(instance); err != nil {
+		if !apierrors.IsNotFound(err) {
+			return false, err
+		}
+		spec = nil
+	} else {
+		spec = &replicaCR.Spec.InstanceSpec
+	}
+	return oc.cleanupOrphanedRuntimeInstance(orphan, instance, longhorn.InstanceManagerTypeReplica, deType, spec)
+}
+
+func (oc *OrphanController) cleanupOrphanedRuntimeInstance(orphan *longhorn.Orphan, instance string, imType longhorn.InstanceManagerType, deType longhorn.DataEngineType, instanceCRSpec *longhorn.InstanceSpec) (isCleanupComplete bool, err error) {
+	if instanceCRSpec != nil && instanceCRSpec.NodeID == oc.controllerID {
+		// Instance is scheduled back to current node and is no longer considered orphaned.
+		// Finalize the orphan CR directly without terminating the instance process.
+		return true, nil
+	}
+
+	im, err := oc.ds.GetRunningInstanceManagerByNodeRO(oc.controllerID, deType)
+	if err != nil {
+		oc.logger.WithError(err).Infof("No running instance manager for node %v for deleting runtime instance orphan %v", oc.controllerID, orphan.Name)
+		return true, nil
+	}
+	imc, err := engineapi.NewInstanceManagerClient(im, false)
+	if err != nil {
+		return false, err
+	}
+	defer func() {
+		if closeErr := imc.Close(); closeErr != nil {
+			oc.logger.WithError(closeErr).Error("failed to close instance manager client")
+		}
+	}()
+
+	runtimeInstanceExist := types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeRuntimeInstanceState).Status == longhorn.ConditionStatusTrue
+	if runtimeInstanceExist {
+		if err = oc.deleteRuntimeInstance(imc, instance, imType, deType); err != nil {
+			return false, err
+		}
+	}
+	return oc.confirmOrphanRuntimeInstanceCleanup(imc, instance, imType, deType)
+}
+
+func (oc *OrphanController) extractOrphanedRuntimeInfo(orphan *longhorn.Orphan) (name string, dataEngineType longhorn.DataEngineType, err error) {
+	name, ok := orphan.Spec.Parameters[longhorn.OrphanInstanceName]
+	if !ok {
+		return "", "", fmt.Errorf("failed to get instance name for runtime instance orphan %v", orphan.Name)
+	}
+
+	dataEngineTypeStr, ok := orphan.Spec.Parameters[longhorn.OrphanDataEngineType]
+	if !ok {
+		return "", "", fmt.Errorf("failed to get data engine type for runtime instance orphan %v", orphan.Name)
+	}
+	switch dataEngineType = longhorn.DataEngineType(dataEngineTypeStr); dataEngineType {
+	case longhorn.DataEngineTypeV1, longhorn.DataEngineTypeV2:
+		// supported data engine type
 	default:
-		err = fmt.Errorf("unknown orphan type %v", orphan.Spec.Type)
+		return "", "", fmt.Errorf("unknown data engine type %v for runtime instance orphan %v", dataEngineTypeStr, orphan.Name)
path: controller/orphan_controller.go
user: login: COLDTURNIP
body: Just add checks for `dataEngineType`. Thanks for reminding.
created_at: 2025-04-18T08:53:58Z
author_association: MEMBER
diff_hunk: @@ -1052,14 +1055,30 @@ var (
 	SettingDefinitionOrphanAutoDeletion = SettingDefinition{
 		DisplayName: "Orphan Auto-Deletion",
 		Description: "This setting allows Longhorn to delete the orphan resource and its corresponding orphaned data automatically. \n\n" +
-			"Orphan resources on down or unknown nodes will not be cleaned up automatically. \n\n",
+			"Orphan resources on down or unknown nodes will not be cleaned up automatically. \n\n" +
path: types/setting.go
user: login: PhanLe1010
body: Is it just setting rename?
created_at: 2025-04-18T23:26:44Z
author_association: NONE
diff_hunk: @@ -55,12 +55,13 @@ type InstanceProcessSpec struct {
 type InstanceState string
 
 const (
-	InstanceStateRunning  = InstanceState("running")
-	InstanceStateStopped  = InstanceState("stopped")
-	InstanceStateError    = InstanceState("error")
-	InstanceStateStarting = InstanceState("starting")
-	InstanceStateStopping = InstanceState("stopping")
-	InstanceStateUnknown  = InstanceState("unknown")
+	InstanceStateRunning    = InstanceState("running")
+	InstanceStateStopped    = InstanceState("stopped")
+	InstanceStateError      = InstanceState("error")
+	InstanceStateStarting   = InstanceState("starting")
+	InstanceStateStopping   = InstanceState("stopping")
+	InstanceStateUnknown    = InstanceState("unknown")
+	InstanceStateTerminated = InstanceState("terminated")
path: k8s/pkg/apis/longhorn/v1beta2/instancemanager.go
user: login: PhanLe1010
body: Could you elaborate more about the use of the InstanceStateTerminated state? 
created_at: 2025-04-18T23:28:12Z
author_association: NONE
diff_hunk: @@ -1052,14 +1055,30 @@ var (
 	SettingDefinitionOrphanAutoDeletion = SettingDefinition{
 		DisplayName: "Orphan Auto-Deletion",
 		Description: "This setting allows Longhorn to delete the orphan resource and its corresponding orphaned data automatically. \n\n" +
-			"Orphan resources on down or unknown nodes will not be cleaned up automatically. \n\n",
+			"Orphan resources on down or unknown nodes will not be cleaned up automatically. \n\n" +
path: types/setting.go
user: login: PhanLe1010
body: Oh. I see. New setting can control multiple orphan types
created_at: 2025-04-18T23:43:38Z
author_association: NONE
diff_hunk: @@ -896,6 +900,32 @@ func (sc *SettingController) syncDefaultLonghornStaticStorageClass() error {
 	return err
 }
 
+func (sc *SettingController) syncOrphanResourceAutoDeletion() error {
+	setting, err := sc.ds.GetSettingOrphanResourceAutoDeletion()
+	if err != nil {
+		return err
+	}
+	var replicaDataCleanupValue string
+	if setting[types.OrphanResourceTypeReplicaData] {
+		replicaDataCleanupValue = "true"
+	} else {
+		replicaDataCleanupValue = "false"
+	}
+
+	deprecatedSetting, err := sc.ds.GetSetting(types.SettingNameOrphanAutoDeletion)
+	if err != nil {
+		return err
+	}
+	if replicaDataCleanupValue != deprecatedSetting.Value {
+		deprecatedSetting.Value = replicaDataCleanupValue
+		if _, err := sc.ds.UpdateSetting(deprecatedSetting); err != nil {
+			return err
+		}
+	}
path: controller/setting_controller.go
user: login: PhanLe1010
body: Do we need to update the deprecatedSetting? Will the deprecated setting value be reset the next time user apply the Longhorn deployment yaml?
created_at: 2025-04-18T23:46:18Z
author_association: NONE
diff_hunk: @@ -1402,7 +1405,7 @@ func (nc *NodeController) deleteOrphans(node *longhorn.Node, diskName string, di
 	return nil
 }
 
-func (nc *NodeController) createOrphans(node *longhorn.Node, diskName string, diskInfo *monitor.CollectedDiskInfo, newOrphanedReplicaDataStores map[string]string) error {
+func (nc *NodeController) createDataStoreOrphans(node *longhorn.Node, diskName string, diskInfo *monitor.CollectedDiskInfo, newOrphanedReplicaDataStores map[string]string) error {
path: controller/node_controller.go
user: login: PhanLe1010
body: Can you elaborate more on why we rename to `createDataStoreOrphans`? What does DataStore mean in this context?
created_at: 2025-04-19T00:20:04Z
author_association: NONE
diff_hunk: @@ -2044,6 +2087,284 @@ func (m *InstanceManagerMonitor) StopMonitorWithLock() {
 	m.done = true
 }
 
+func (m *InstanceManagerMonitor) syncOrphans(im *longhorn.InstanceManager, instanceProcess map[string]longhorn.InstanceProcess) {
+	engineProcess := map[string]longhorn.InstanceProcess{}
+	replicaProcess := map[string]longhorn.InstanceProcess{}
+	for name, process := range instanceProcess {
+		switch process.Status.Type {
+		case longhorn.InstanceTypeEngine:
+			engineProcess[name] = process
+		case longhorn.InstanceTypeReplica:
+			replicaProcess[name] = process
+		}
+	}
+
+	existOrphans, err := m.getNodeRuntimeOrphanMap()
+	if err != nil {
+		m.logger.WithError(err).Errorf("Failed to list orphans on node %s", m.controllerID)
+		return
+	}
+
+	// create orphan CR for orphaned instances
+	for _, engineInstance := range engineProcess {
+		if isOrphaned, err := m.isEngineOrphaned(engineInstance.Spec.Name); err != nil {
+			m.logger.WithError(err).Errorf("Failed to check orphan for engine instance %v", engineInstance.Spec.Name)
+		} else if isOrphaned {
+			m.logger.Infof("Engine instance %v is orphaned", engineInstance.Spec.Name)
+			err := m.createOrphan(im, engineInstance.Spec.Name, longhorn.OrphanTypeEngineInstance, engineInstance.Spec.DataEngine)
+			if err != nil {
+				m.logger.WithError(err).Errorf("Failed to create orphan for engine instance %v", engineInstance.Spec.Name)
+			}
+		}
+	}
+	for _, replicaInstance := range replicaProcess {
+		if isOrphaned, err := m.isReplicaOrphaned(replicaInstance.Spec.Name); err != nil {
+			m.logger.WithError(err).Errorf("Failed to check orphan for replica instance %v", replicaInstance.Spec.Name)
+		} else if isOrphaned {
+			m.logger.Infof("Replica instance %v is orphaned", replicaInstance.Spec.Name)
+			err := m.createOrphan(im, replicaInstance.Spec.Name, longhorn.OrphanTypeReplicaInstance, replicaInstance.Spec.DataEngine)
+			if err != nil {
+				m.logger.WithError(err).Errorf("Failed to create orphan for replica instance %v", replicaInstance.Spec.Name)
+			}
+		}
+	}
+
+	// update instance state on orphan CRs
+	deleteOrphans := make(map[string]bool)
+	for orphanName, orphan := range existOrphans {
+		instanceName := orphan.Spec.Parameters[longhorn.OrphanInstanceName]
+		var instanceProc longhorn.InstanceProcess
+		var instanceExist, instanceCRScheduledBack bool
+		switch orphan.Spec.Type {
+		case longhorn.OrphanTypeEngineInstance:
+			instanceProc, instanceExist = engineProcess[instanceName]
+			isCRScheduledBack, err := m.isEngineScheduledOnNode(instanceName)
+			if err != nil {
+				m.logger.WithError(err).WithField("instance", instanceName).Errorf("Failed to check CR scheduled node for engine instance orphan %v", orphanName)
+				instanceCRScheduledBack = false
+			} else {
+				instanceCRScheduledBack = isCRScheduledBack
+			}
+		case longhorn.OrphanTypeReplicaInstance:
+			instanceProc, instanceExist = replicaProcess[instanceName]
+			isCRScheduledBack, err := m.isReplicaScheduledOnNode(instanceName)
+			if err != nil {
+				m.logger.WithError(err).WithField("instance", instanceName).Errorf("Failed to check CR scheduled node for replica instance orphan %v", orphanName)
+				instanceCRScheduledBack = false
+			} else {
+				instanceCRScheduledBack = isCRScheduledBack
+			}
+		}
+		var instanceState longhorn.InstanceState
+		if instanceExist {
+			instanceState = instanceProc.Status.State
+		} else {
+			instanceState = longhorn.InstanceStateTerminated
+		}
+		if updateErr := m.updateOrphanInstanceStatus(orphan, instanceState); updateErr != nil {
+			m.logger.WithError(updateErr).WithField("instanceState", instanceState).Errorf("Failed to update instance state for orphan %s", orphanName)
+		}
+
+		if orphan.DeletionTimestamp.IsZero() {
+			if !instanceExist || instanceCRScheduledBack {
+				m.logger.WithFields(logrus.Fields{
+					"instanceExist":           instanceExist,
+					"instanceState":           instanceState,
+					"instanceCRScheduledBack": instanceCRScheduledBack,
+				}).Debugf("Orphan instance %s is deletable", orphanName)
+				deleteOrphans[orphanName] = true
+			}
+		}
+	}
+
+	// delete orphan CRs
+	autoDeletionTypes, err := m.ds.GetSettingOrphanResourceAutoDeletion()
+	var autoDeleteEngine = false
+	var autoDeleteReplica = false
+	if err != nil {
+		m.logger.WithError(err).Warnf("Failed to fetch orphan auto deletion setting, disable by default")
+	} else {
+		autoDeleteEngine = autoDeletionTypes[types.OrphanResourceTypeEngineInstance]
+		autoDeleteReplica = autoDeletionTypes[types.OrphanResourceTypeReplicaInstance]
+	}
+	m.cleanupOrphans(existOrphans, deleteOrphans, autoDeleteEngine, autoDeleteReplica)
+}
+
+func (m *InstanceManagerMonitor) getNodeRuntimeOrphanMap() (map[string]*longhorn.Orphan, error) {
+	existOrphans, err := m.ds.ListOrphansByNode(m.controllerID)
+	if err != nil {
+		return nil, err
+	}
+	orphanMap := make(map[string]*longhorn.Orphan)
+	for _, orphan := range existOrphans {
+		switch orphan.Spec.Type {
+		case longhorn.OrphanTypeEngineInstance, longhorn.OrphanTypeReplicaInstance:
+			orphanMap[orphan.Name] = orphan
+		}
+	}
+	return orphanMap, nil
+}
+
+func (m *InstanceManagerMonitor) isEngineOrphaned(instanceName string) (bool, error) {
+	existEngine, err := m.ds.GetEngineRO(instanceName)
+	switch {
+	case err == nil:
+		// Engine CR still exists - check the ownership
+		return m.isInstanceOrphanedOnNode(&existEngine.Spec.InstanceSpec, &existEngine.Status.InstanceStatus), nil
+	case apierrors.IsNotFound(err):
+		// Engine CR not found - instance is orphaned
+		return true, nil
+	default:
+		// Unexpected error - unable to check if engine instance is orphaned or not
+		return false, err
+	}
+}
+
+func (m *InstanceManagerMonitor) isReplicaOrphaned(instanceName string) (bool, error) {
+	existReplica, err := m.ds.GetReplicaRO(instanceName)
+	switch {
+	case err == nil:
+		// Replica CR still exists - check the ownership
+		return m.isInstanceOrphanedOnNode(&existReplica.Spec.InstanceSpec, &existReplica.Status.InstanceStatus), nil
+	case apierrors.IsNotFound(err):
+		// Replica CR not found - instance is orphaned
+		return true, nil
+	default:
+		// Unexpected error - unable to check if replica instance is orphaned or not
+		return false, err
+	}
+}
+
+// isInstanceOrphanedOnNode returns true only when it is very certain that an instance is scheduled on another node
+func (m *InstanceManagerMonitor) isInstanceOrphanedOnNode(spec *longhorn.InstanceSpec, status *longhorn.InstanceStatus) bool {
+	if status.CurrentState != spec.DesireState {
+		return false
+	}
+	switch status.CurrentState {
+	case longhorn.InstanceStateRunning:
+		return status.OwnerID == spec.NodeID && spec.NodeID != m.controllerID
+	case longhorn.InstanceStateStopped:
+		return spec.NodeID != m.controllerID
+	default:
+		return false
+	}
+}
+
+func (m *InstanceManagerMonitor) isEngineScheduledOnNode(instanceName string) (bool, error) {
+	existEngine, err := m.ds.GetEngineRO(instanceName)
+	switch {
+	case err == nil:
+		// Engine CR still exists - check the ownership
+		return m.isInstanceScheduledOnNode(&existEngine.Spec.InstanceSpec, &existEngine.Status.InstanceStatus), nil
+	case apierrors.IsNotFound(err):
+		// Engine CR not found - instance is orphaned
+		return false, nil
+	default:
+		// Unexpected error - unable to check if engine instance is orphaned or not
+		return false, err
+	}
+}
+
+func (m *InstanceManagerMonitor) isReplicaScheduledOnNode(instanceName string) (bool, error) {
+	existReplica, err := m.ds.GetReplicaRO(instanceName)
+	switch {
+	case err == nil:
+		// Replica CR still exists - check the ownership
+		return m.isInstanceScheduledOnNode(&existReplica.Spec.InstanceSpec, &existReplica.Status.InstanceStatus), nil
+	case apierrors.IsNotFound(err):
+		// Replica CR not found - instance is orphaned
+		return false, nil
+	default:
+		// Unexpected error - unable to check if replica instance is orphaned or not
+		return false, err
+	}
+}
+
+// isInstanceScheduledOnNode returns true only when it is very certain that an instance is scheduled on this node
+func (m *InstanceManagerMonitor) isInstanceScheduledOnNode(spec *longhorn.InstanceSpec, status *longhorn.InstanceStatus) bool {
+	if status.CurrentState != spec.DesireState {
+		return false
+	}
+	switch status.CurrentState {
+	case longhorn.InstanceStateRunning:
+		return status.OwnerID == spec.NodeID && spec.NodeID == m.controllerID
+	case longhorn.InstanceStateStopped:
+		return spec.NodeID == m.controllerID
+	default:
+		return false
+	}
+}
+
+func (m *InstanceManagerMonitor) createOrphan(im *longhorn.InstanceManager, instanceName string, orphanType longhorn.OrphanType, dataEngineType longhorn.DataEngineType) error {
+	name := types.GetOrphanChecksumNameForOrphanedInstance(instanceName, m.controllerID, string(dataEngineType))
+
+	if _, err := m.ds.GetOrphanRO(name); err == nil || !apierrors.IsNotFound(err) {
+		return err
+	}
+
+	// labels will be attached by mutator webhook
+	orphan := &longhorn.Orphan{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:            name,
+			OwnerReferences: datastore.GetOwnerReferencesForInstanceManager(im),
+		},
+		Spec: longhorn.OrphanSpec{
+			NodeID: m.controllerID,
+			Type:   orphanType,
+			Parameters: map[string]string{
+				longhorn.OrphanInstanceName:   instanceName,
+				longhorn.OrphanDataEngineType: string(dataEngineType),
+			},
+		},
+	}
+
+	_, err := m.ds.CreateOrphan(orphan)
+	return err
+}
+
+func (m *InstanceManagerMonitor) updateOrphanInstanceStatus(orphan *longhorn.Orphan, instanceState longhorn.InstanceState) error {
path: controller/instance_manager_controller.go
user: login: PhanLe1010
body: Updating orphan CR status inside instance manager controller doesn't appropriate according to the general rule of k8s controller. AK.A only update the status of the CR inside its controller 

WDYT @derekbit 
created_at: 2025-04-19T00:31:17Z
author_association: NONE
diff_hunk: @@ -2044,6 +2087,284 @@ func (m *InstanceManagerMonitor) StopMonitorWithLock() {
 	m.done = true
 }
 
+func (m *InstanceManagerMonitor) syncOrphans(im *longhorn.InstanceManager, instanceProcess map[string]longhorn.InstanceProcess) {
+	engineProcess := map[string]longhorn.InstanceProcess{}
+	replicaProcess := map[string]longhorn.InstanceProcess{}
+	for name, process := range instanceProcess {
+		switch process.Status.Type {
+		case longhorn.InstanceTypeEngine:
+			engineProcess[name] = process
+		case longhorn.InstanceTypeReplica:
+			replicaProcess[name] = process
+		}
+	}
+
+	existOrphans, err := m.getNodeRuntimeOrphanMap()
+	if err != nil {
+		m.logger.WithError(err).Errorf("Failed to list orphans on node %s", m.controllerID)
+		return
+	}
+
+	// create orphan CR for orphaned instances
+	for _, engineInstance := range engineProcess {
+		if isOrphaned, err := m.isEngineOrphaned(engineInstance.Spec.Name); err != nil {
+			m.logger.WithError(err).Errorf("Failed to check orphan for engine instance %v", engineInstance.Spec.Name)
+		} else if isOrphaned {
+			m.logger.Infof("Engine instance %v is orphaned", engineInstance.Spec.Name)
+			err := m.createOrphan(im, engineInstance.Spec.Name, longhorn.OrphanTypeEngineInstance, engineInstance.Spec.DataEngine)
+			if err != nil {
+				m.logger.WithError(err).Errorf("Failed to create orphan for engine instance %v", engineInstance.Spec.Name)
+			}
+		}
+	}
+	for _, replicaInstance := range replicaProcess {
+		if isOrphaned, err := m.isReplicaOrphaned(replicaInstance.Spec.Name); err != nil {
+			m.logger.WithError(err).Errorf("Failed to check orphan for replica instance %v", replicaInstance.Spec.Name)
+		} else if isOrphaned {
+			m.logger.Infof("Replica instance %v is orphaned", replicaInstance.Spec.Name)
+			err := m.createOrphan(im, replicaInstance.Spec.Name, longhorn.OrphanTypeReplicaInstance, replicaInstance.Spec.DataEngine)
+			if err != nil {
+				m.logger.WithError(err).Errorf("Failed to create orphan for replica instance %v", replicaInstance.Spec.Name)
+			}
+		}
+	}
+
+	// update instance state on orphan CRs
+	deleteOrphans := make(map[string]bool)
+	for orphanName, orphan := range existOrphans {
+		instanceName := orphan.Spec.Parameters[longhorn.OrphanInstanceName]
+		var instanceProc longhorn.InstanceProcess
+		var instanceExist, instanceCRScheduledBack bool
+		switch orphan.Spec.Type {
+		case longhorn.OrphanTypeEngineInstance:
+			instanceProc, instanceExist = engineProcess[instanceName]
+			isCRScheduledBack, err := m.isEngineScheduledOnNode(instanceName)
+			if err != nil {
+				m.logger.WithError(err).WithField("instance", instanceName).Errorf("Failed to check CR scheduled node for engine instance orphan %v", orphanName)
+				instanceCRScheduledBack = false
+			} else {
+				instanceCRScheduledBack = isCRScheduledBack
+			}
+		case longhorn.OrphanTypeReplicaInstance:
+			instanceProc, instanceExist = replicaProcess[instanceName]
+			isCRScheduledBack, err := m.isReplicaScheduledOnNode(instanceName)
+			if err != nil {
+				m.logger.WithError(err).WithField("instance", instanceName).Errorf("Failed to check CR scheduled node for replica instance orphan %v", orphanName)
+				instanceCRScheduledBack = false
+			} else {
+				instanceCRScheduledBack = isCRScheduledBack
+			}
+		}
+		var instanceState longhorn.InstanceState
+		if instanceExist {
+			instanceState = instanceProc.Status.State
+		} else {
+			instanceState = longhorn.InstanceStateTerminated
+		}
+		if updateErr := m.updateOrphanInstanceStatus(orphan, instanceState); updateErr != nil {
+			m.logger.WithError(updateErr).WithField("instanceState", instanceState).Errorf("Failed to update instance state for orphan %s", orphanName)
+		}
+
+		if orphan.DeletionTimestamp.IsZero() {
+			if !instanceExist || instanceCRScheduledBack {
+				m.logger.WithFields(logrus.Fields{
+					"instanceExist":           instanceExist,
+					"instanceState":           instanceState,
+					"instanceCRScheduledBack": instanceCRScheduledBack,
+				}).Debugf("Orphan instance %s is deletable", orphanName)
+				deleteOrphans[orphanName] = true
+			}
+		}
+	}
+
+	// delete orphan CRs
+	autoDeletionTypes, err := m.ds.GetSettingOrphanResourceAutoDeletion()
+	var autoDeleteEngine = false
+	var autoDeleteReplica = false
+	if err != nil {
+		m.logger.WithError(err).Warnf("Failed to fetch orphan auto deletion setting, disable by default")
+	} else {
+		autoDeleteEngine = autoDeletionTypes[types.OrphanResourceTypeEngineInstance]
+		autoDeleteReplica = autoDeletionTypes[types.OrphanResourceTypeReplicaInstance]
+	}
+	m.cleanupOrphans(existOrphans, deleteOrphans, autoDeleteEngine, autoDeleteReplica)
+}
+
+func (m *InstanceManagerMonitor) getNodeRuntimeOrphanMap() (map[string]*longhorn.Orphan, error) {
+	existOrphans, err := m.ds.ListOrphansByNode(m.controllerID)
+	if err != nil {
+		return nil, err
+	}
+	orphanMap := make(map[string]*longhorn.Orphan)
+	for _, orphan := range existOrphans {
+		switch orphan.Spec.Type {
+		case longhorn.OrphanTypeEngineInstance, longhorn.OrphanTypeReplicaInstance:
+			orphanMap[orphan.Name] = orphan
+		}
+	}
+	return orphanMap, nil
+}
+
+func (m *InstanceManagerMonitor) isEngineOrphaned(instanceName string) (bool, error) {
+	existEngine, err := m.ds.GetEngineRO(instanceName)
+	switch {
+	case err == nil:
+		// Engine CR still exists - check the ownership
+		return m.isInstanceOrphanedOnNode(&existEngine.Spec.InstanceSpec, &existEngine.Status.InstanceStatus), nil
+	case apierrors.IsNotFound(err):
+		// Engine CR not found - instance is orphaned
+		return true, nil
+	default:
+		// Unexpected error - unable to check if engine instance is orphaned or not
+		return false, err
+	}
+}
+
+func (m *InstanceManagerMonitor) isReplicaOrphaned(instanceName string) (bool, error) {
+	existReplica, err := m.ds.GetReplicaRO(instanceName)
+	switch {
+	case err == nil:
+		// Replica CR still exists - check the ownership
+		return m.isInstanceOrphanedOnNode(&existReplica.Spec.InstanceSpec, &existReplica.Status.InstanceStatus), nil
+	case apierrors.IsNotFound(err):
+		// Replica CR not found - instance is orphaned
+		return true, nil
+	default:
+		// Unexpected error - unable to check if replica instance is orphaned or not
+		return false, err
+	}
+}
+
+// isInstanceOrphanedOnNode returns true only when it is very certain that an instance is scheduled on another node
+func (m *InstanceManagerMonitor) isInstanceOrphanedOnNode(spec *longhorn.InstanceSpec, status *longhorn.InstanceStatus) bool {
+	if status.CurrentState != spec.DesireState {
+		return false
+	}
+	switch status.CurrentState {
+	case longhorn.InstanceStateRunning:
+		return status.OwnerID == spec.NodeID && spec.NodeID != m.controllerID
+	case longhorn.InstanceStateStopped:
+		return spec.NodeID != m.controllerID
+	default:
+		return false
+	}
+}
+
+func (m *InstanceManagerMonitor) isEngineScheduledOnNode(instanceName string) (bool, error) {
+	existEngine, err := m.ds.GetEngineRO(instanceName)
+	switch {
+	case err == nil:
+		// Engine CR still exists - check the ownership
+		return m.isInstanceScheduledOnNode(&existEngine.Spec.InstanceSpec, &existEngine.Status.InstanceStatus), nil
+	case apierrors.IsNotFound(err):
+		// Engine CR not found - instance is orphaned
+		return false, nil
+	default:
+		// Unexpected error - unable to check if engine instance is orphaned or not
+		return false, err
+	}
+}
+
+func (m *InstanceManagerMonitor) isReplicaScheduledOnNode(instanceName string) (bool, error) {
+	existReplica, err := m.ds.GetReplicaRO(instanceName)
+	switch {
+	case err == nil:
+		// Replica CR still exists - check the ownership
+		return m.isInstanceScheduledOnNode(&existReplica.Spec.InstanceSpec, &existReplica.Status.InstanceStatus), nil
+	case apierrors.IsNotFound(err):
+		// Replica CR not found - instance is orphaned
+		return false, nil
+	default:
+		// Unexpected error - unable to check if replica instance is orphaned or not
+		return false, err
+	}
+}
+
+// isInstanceScheduledOnNode returns true only when it is very certain that an instance is scheduled on this node
+func (m *InstanceManagerMonitor) isInstanceScheduledOnNode(spec *longhorn.InstanceSpec, status *longhorn.InstanceStatus) bool {
+	if status.CurrentState != spec.DesireState {
+		return false
+	}
+	switch status.CurrentState {
+	case longhorn.InstanceStateRunning:
+		return status.OwnerID == spec.NodeID && spec.NodeID == m.controllerID
+	case longhorn.InstanceStateStopped:
+		return spec.NodeID == m.controllerID
+	default:
+		return false
+	}
+}
+
+func (m *InstanceManagerMonitor) createOrphan(im *longhorn.InstanceManager, instanceName string, orphanType longhorn.OrphanType, dataEngineType longhorn.DataEngineType) error {
+	name := types.GetOrphanChecksumNameForOrphanedInstance(instanceName, m.controllerID, string(dataEngineType))
+
+	if _, err := m.ds.GetOrphanRO(name); err == nil || !apierrors.IsNotFound(err) {
+		return err
+	}
+
+	// labels will be attached by mutator webhook
+	orphan := &longhorn.Orphan{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:            name,
+			OwnerReferences: datastore.GetOwnerReferencesForInstanceManager(im),
+		},
+		Spec: longhorn.OrphanSpec{
+			NodeID: m.controllerID,
+			Type:   orphanType,
+			Parameters: map[string]string{
+				longhorn.OrphanInstanceName:   instanceName,
+				longhorn.OrphanDataEngineType: string(dataEngineType),
+			},
+		},
+	}
+
+	_, err := m.ds.CreateOrphan(orphan)
+	return err
+}
+
+func (m *InstanceManagerMonitor) updateOrphanInstanceStatus(orphan *longhorn.Orphan, instanceState longhorn.InstanceState) error {
path: controller/instance_manager_controller.go
user: login: derekbit
body: Agree. We should prevent the anti-pattern behavior.
created_at: 2025-04-20T05:28:13Z
author_association: MEMBER
diff_hunk: @@ -2044,6 +2087,284 @@ func (m *InstanceManagerMonitor) StopMonitorWithLock() {
 	m.done = true
 }
 
+func (m *InstanceManagerMonitor) syncOrphans(im *longhorn.InstanceManager, instanceProcess map[string]longhorn.InstanceProcess) {
+	engineProcess := map[string]longhorn.InstanceProcess{}
+	replicaProcess := map[string]longhorn.InstanceProcess{}
+	for name, process := range instanceProcess {
+		switch process.Status.Type {
+		case longhorn.InstanceTypeEngine:
+			engineProcess[name] = process
+		case longhorn.InstanceTypeReplica:
+			replicaProcess[name] = process
+		}
+	}
+
+	existOrphans, err := m.getNodeRuntimeOrphanMap()
+	if err != nil {
+		m.logger.WithError(err).Errorf("Failed to list orphans on node %s", m.controllerID)
+		return
+	}
+
+	// create orphan CR for orphaned instances
+	for _, engineInstance := range engineProcess {
+		if isOrphaned, err := m.isEngineOrphaned(engineInstance.Spec.Name); err != nil {
+			m.logger.WithError(err).Errorf("Failed to check orphan for engine instance %v", engineInstance.Spec.Name)
+		} else if isOrphaned {
+			m.logger.Infof("Engine instance %v is orphaned", engineInstance.Spec.Name)
+			err := m.createOrphan(im, engineInstance.Spec.Name, longhorn.OrphanTypeEngineInstance, engineInstance.Spec.DataEngine)
+			if err != nil {
+				m.logger.WithError(err).Errorf("Failed to create orphan for engine instance %v", engineInstance.Spec.Name)
+			}
+		}
+	}
+	for _, replicaInstance := range replicaProcess {
+		if isOrphaned, err := m.isReplicaOrphaned(replicaInstance.Spec.Name); err != nil {
+			m.logger.WithError(err).Errorf("Failed to check orphan for replica instance %v", replicaInstance.Spec.Name)
+		} else if isOrphaned {
+			m.logger.Infof("Replica instance %v is orphaned", replicaInstance.Spec.Name)
+			err := m.createOrphan(im, replicaInstance.Spec.Name, longhorn.OrphanTypeReplicaInstance, replicaInstance.Spec.DataEngine)
+			if err != nil {
+				m.logger.WithError(err).Errorf("Failed to create orphan for replica instance %v", replicaInstance.Spec.Name)
+			}
+		}
+	}
+
+	// update instance state on orphan CRs
+	deleteOrphans := make(map[string]bool)
+	for orphanName, orphan := range existOrphans {
+		instanceName := orphan.Spec.Parameters[longhorn.OrphanInstanceName]
+		var instanceProc longhorn.InstanceProcess
+		var instanceExist, instanceCRScheduledBack bool
+		switch orphan.Spec.Type {
+		case longhorn.OrphanTypeEngineInstance:
+			instanceProc, instanceExist = engineProcess[instanceName]
+			isCRScheduledBack, err := m.isEngineScheduledOnNode(instanceName)
+			if err != nil {
+				m.logger.WithError(err).WithField("instance", instanceName).Errorf("Failed to check CR scheduled node for engine instance orphan %v", orphanName)
+				instanceCRScheduledBack = false
+			} else {
+				instanceCRScheduledBack = isCRScheduledBack
+			}
+		case longhorn.OrphanTypeReplicaInstance:
+			instanceProc, instanceExist = replicaProcess[instanceName]
+			isCRScheduledBack, err := m.isReplicaScheduledOnNode(instanceName)
+			if err != nil {
+				m.logger.WithError(err).WithField("instance", instanceName).Errorf("Failed to check CR scheduled node for replica instance orphan %v", orphanName)
+				instanceCRScheduledBack = false
+			} else {
+				instanceCRScheduledBack = isCRScheduledBack
+			}
+		}
+		var instanceState longhorn.InstanceState
+		if instanceExist {
+			instanceState = instanceProc.Status.State
+		} else {
+			instanceState = longhorn.InstanceStateTerminated
+		}
+		if updateErr := m.updateOrphanInstanceStatus(orphan, instanceState); updateErr != nil {
+			m.logger.WithError(updateErr).WithField("instanceState", instanceState).Errorf("Failed to update instance state for orphan %s", orphanName)
+		}
+
+		if orphan.DeletionTimestamp.IsZero() {
+			if !instanceExist || instanceCRScheduledBack {
+				m.logger.WithFields(logrus.Fields{
+					"instanceExist":           instanceExist,
+					"instanceState":           instanceState,
+					"instanceCRScheduledBack": instanceCRScheduledBack,
+				}).Debugf("Orphan instance %s is deletable", orphanName)
+				deleteOrphans[orphanName] = true
+			}
+		}
+	}
+
+	// delete orphan CRs
+	autoDeletionTypes, err := m.ds.GetSettingOrphanResourceAutoDeletion()
+	var autoDeleteEngine = false
+	var autoDeleteReplica = false
+	if err != nil {
+		m.logger.WithError(err).Warnf("Failed to fetch orphan auto deletion setting, disable by default")
+	} else {
+		autoDeleteEngine = autoDeletionTypes[types.OrphanResourceTypeEngineInstance]
+		autoDeleteReplica = autoDeletionTypes[types.OrphanResourceTypeReplicaInstance]
+	}
+	m.cleanupOrphans(existOrphans, deleteOrphans, autoDeleteEngine, autoDeleteReplica)
+}
+
+func (m *InstanceManagerMonitor) getNodeRuntimeOrphanMap() (map[string]*longhorn.Orphan, error) {
+	existOrphans, err := m.ds.ListOrphansByNode(m.controllerID)
+	if err != nil {
+		return nil, err
+	}
+	orphanMap := make(map[string]*longhorn.Orphan)
+	for _, orphan := range existOrphans {
+		switch orphan.Spec.Type {
+		case longhorn.OrphanTypeEngineInstance, longhorn.OrphanTypeReplicaInstance:
+			orphanMap[orphan.Name] = orphan
+		}
+	}
+	return orphanMap, nil
+}
+
+func (m *InstanceManagerMonitor) isEngineOrphaned(instanceName string) (bool, error) {
+	existEngine, err := m.ds.GetEngineRO(instanceName)
+	switch {
+	case err == nil:
+		// Engine CR still exists - check the ownership
+		return m.isInstanceOrphanedOnNode(&existEngine.Spec.InstanceSpec, &existEngine.Status.InstanceStatus), nil
+	case apierrors.IsNotFound(err):
+		// Engine CR not found - instance is orphaned
+		return true, nil
+	default:
+		// Unexpected error - unable to check if engine instance is orphaned or not
+		return false, err
+	}
+}
+
+func (m *InstanceManagerMonitor) isReplicaOrphaned(instanceName string) (bool, error) {
+	existReplica, err := m.ds.GetReplicaRO(instanceName)
+	switch {
+	case err == nil:
+		// Replica CR still exists - check the ownership
+		return m.isInstanceOrphanedOnNode(&existReplica.Spec.InstanceSpec, &existReplica.Status.InstanceStatus), nil
+	case apierrors.IsNotFound(err):
+		// Replica CR not found - instance is orphaned
+		return true, nil
+	default:
+		// Unexpected error - unable to check if replica instance is orphaned or not
+		return false, err
+	}
+}
+
+// isInstanceOrphanedOnNode returns true only when it is very certain that an instance is scheduled on another node
+func (m *InstanceManagerMonitor) isInstanceOrphanedOnNode(spec *longhorn.InstanceSpec, status *longhorn.InstanceStatus) bool {
+	if status.CurrentState != spec.DesireState {
+		return false
+	}
+	switch status.CurrentState {
+	case longhorn.InstanceStateRunning:
+		return status.OwnerID == spec.NodeID && spec.NodeID != m.controllerID
+	case longhorn.InstanceStateStopped:
+		return spec.NodeID != m.controllerID
+	default:
+		return false
+	}
+}
+
+func (m *InstanceManagerMonitor) isEngineScheduledOnNode(instanceName string) (bool, error) {
+	existEngine, err := m.ds.GetEngineRO(instanceName)
+	switch {
+	case err == nil:
+		// Engine CR still exists - check the ownership
+		return m.isInstanceScheduledOnNode(&existEngine.Spec.InstanceSpec, &existEngine.Status.InstanceStatus), nil
+	case apierrors.IsNotFound(err):
+		// Engine CR not found - instance is orphaned
+		return false, nil
+	default:
+		// Unexpected error - unable to check if engine instance is orphaned or not
+		return false, err
+	}
+}
+
+func (m *InstanceManagerMonitor) isReplicaScheduledOnNode(instanceName string) (bool, error) {
+	existReplica, err := m.ds.GetReplicaRO(instanceName)
+	switch {
+	case err == nil:
+		// Replica CR still exists - check the ownership
+		return m.isInstanceScheduledOnNode(&existReplica.Spec.InstanceSpec, &existReplica.Status.InstanceStatus), nil
+	case apierrors.IsNotFound(err):
+		// Replica CR not found - instance is orphaned
+		return false, nil
+	default:
+		// Unexpected error - unable to check if replica instance is orphaned or not
+		return false, err
+	}
+}
+
+// isInstanceScheduledOnNode returns true only when it is very certain that an instance is scheduled on this node
+func (m *InstanceManagerMonitor) isInstanceScheduledOnNode(spec *longhorn.InstanceSpec, status *longhorn.InstanceStatus) bool {
+	if status.CurrentState != spec.DesireState {
+		return false
+	}
+	switch status.CurrentState {
+	case longhorn.InstanceStateRunning:
+		return status.OwnerID == spec.NodeID && spec.NodeID == m.controllerID
+	case longhorn.InstanceStateStopped:
+		return spec.NodeID == m.controllerID
+	default:
+		return false
+	}
+}
+
+func (m *InstanceManagerMonitor) createOrphan(im *longhorn.InstanceManager, instanceName string, orphanType longhorn.OrphanType, dataEngineType longhorn.DataEngineType) error {
+	name := types.GetOrphanChecksumNameForOrphanedInstance(instanceName, m.controllerID, string(dataEngineType))
+
+	if _, err := m.ds.GetOrphanRO(name); err == nil || !apierrors.IsNotFound(err) {
+		return err
+	}
+
+	// labels will be attached by mutator webhook
+	orphan := &longhorn.Orphan{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:            name,
+			OwnerReferences: datastore.GetOwnerReferencesForInstanceManager(im),
+		},
+		Spec: longhorn.OrphanSpec{
+			NodeID: m.controllerID,
+			Type:   orphanType,
+			Parameters: map[string]string{
+				longhorn.OrphanInstanceName:   instanceName,
+				longhorn.OrphanDataEngineType: string(dataEngineType),
+			},
+		},
+	}
+
+	_, err := m.ds.CreateOrphan(orphan)
+	return err
+}
+
+func (m *InstanceManagerMonitor) updateOrphanInstanceStatus(orphan *longhorn.Orphan, instanceState longhorn.InstanceState) error {
path: controller/instance_manager_controller.go
user: login: COLDTURNIP
body: Thanks, will move instance status update to orphan controller when it reconciling instance manager status update.
created_at: 2025-04-21T01:21:25Z
author_association: MEMBER
diff_hunk: @@ -20,6 +23,9 @@ const (
 )
 
 const (
+	OrphanInstanceName   = "InstanceName"
+	OrphanDataEngineType = "DataEngineType"
path: k8s/pkg/apis/longhorn/v1beta2/orphan.go
user: login: derekbit
body: Could we add a field `DataEngine` in OrphanSpec instead?
created_at: 2025-04-21T01:26:35Z
author_association: MEMBER
diff_hunk: @@ -1402,7 +1405,7 @@ func (nc *NodeController) deleteOrphans(node *longhorn.Node, diskName string, di
 	return nil
 }
 
-func (nc *NodeController) createOrphans(node *longhorn.Node, diskName string, diskInfo *monitor.CollectedDiskInfo, newOrphanedReplicaDataStores map[string]string) error {
+func (nc *NodeController) createDataStoreOrphans(node *longhorn.Node, diskName string, diskInfo *monitor.CollectedDiskInfo, newOrphanedReplicaDataStores map[string]string) error {
path: controller/node_controller.go
user: login: COLDTURNIP
body: In this PR, besides the existing data store orphan, we introduce new types of orphans for runtime instance. The node controller observes the disk status change, and has no knowledge of engine/replica instances. Rename the method here to emphasize that the node controller will create only the data store orphans.
created_at: 2025-04-21T01:27:13Z
author_association: MEMBER
diff_hunk: @@ -1754,6 +1782,32 @@ func UnmarshalNodeSelector(nodeSelectorSetting string) (map[string]string, error
 	return nodeSelector, nil
 }
 
+func UnmarshalOrphanResourceTypes(resourceTypesSetting string) (map[OrphanResourceType]bool, error) {
+	resourceTypes := map[OrphanResourceType]bool{
+		OrphanResourceTypeReplicaData:     false,
+		OrphanResourceTypeEngineInstance:  false,
+		OrphanResourceTypeReplicaInstance: false,
+	}
+
+	resourceTypesSetting = strings.Trim(resourceTypesSetting, " ")
+	invalidItems := make([]string, 0, len(resourceTypesSetting))
+	if resourceTypesSetting != "" {
+		resourceTypeList := strings.Split(resourceTypesSetting, ";")
+		for _, item := range resourceTypeList {
+			resourceType := OrphanResourceType(strings.Trim(item, " "))
+			if _, ok := resourceTypes[resourceType]; ok {
+				resourceTypes[resourceType] = true
+			} else {
+				invalidItems = append(invalidItems, item)
+			}
+		}
+	}
+	if len(invalidItems) > 0 {
+		return nil, fmt.Errorf("invalid orphan resource types: %s", strings.Join(invalidItems, ", "))
+	}
+	return resourceTypes, nil
+}
path: types/setting.go
user: login: derekbit
body: nit: move to go-common-libs in the future.
created_at: 2025-04-21T01:30:46Z
author_association: MEMBER
diff_hunk: @@ -1316,10 +1316,10 @@ func (nc *NodeController) syncOrphans(node *longhorn.Node, collectedDataInfo map
 		newOrphanedReplicaDataStores, missingOrphanedReplicaDataStores :=
 			nc.getNewAndMissingOrphanedReplicaDataStores(diskName, diskInfo.DiskUUID, diskInfo.Path, diskInfo.OrphanedReplicaDataStores)
 
-		if err := nc.createOrphans(node, diskName, diskInfo, newOrphanedReplicaDataStores); err != nil {
+		if err := nc.createDataStoreOrphans(node, diskName, diskInfo, newOrphanedReplicaDataStores); err != nil {
path: controller/node_controller.go
user: login: derekbit
body: IMO for the convention, we should use the original naming `createOrphans`.
created_at: 2025-04-21T01:33:10Z
author_association: MEMBER
diff_hunk: @@ -1316,10 +1316,10 @@ func (nc *NodeController) syncOrphans(node *longhorn.Node, collectedDataInfo map
 		newOrphanedReplicaDataStores, missingOrphanedReplicaDataStores :=
 			nc.getNewAndMissingOrphanedReplicaDataStores(diskName, diskInfo.DiskUUID, diskInfo.Path, diskInfo.OrphanedReplicaDataStores)
 
-		if err := nc.createOrphans(node, diskName, diskInfo, newOrphanedReplicaDataStores); err != nil {
+		if err := nc.createDataStoreOrphans(node, diskName, diskInfo, newOrphanedReplicaDataStores); err != nil {
 			return errors.Wrapf(err, "failed to create orphans for disk %v", diskName)
 		}
-		if err := nc.deleteOrphans(node, diskName, diskInfo, missingOrphanedReplicaDataStores); err != nil {
+		if err := nc.deleteDataStoreOrphans(node, diskName, diskInfo, missingOrphanedReplicaDataStores); err != nil {
path: controller/node_controller.go
user: login: derekbit
body: Same as above
created_at: 2025-04-21T01:33:18Z
author_association: MEMBER
diff_hunk: @@ -1402,7 +1405,7 @@ func (nc *NodeController) deleteOrphans(node *longhorn.Node, diskName string, di
 	return nil
 }
 
-func (nc *NodeController) createOrphans(node *longhorn.Node, diskName string, diskInfo *monitor.CollectedDiskInfo, newOrphanedReplicaDataStores map[string]string) error {
+func (nc *NodeController) createDataStoreOrphans(node *longhorn.Node, diskName string, diskInfo *monitor.CollectedDiskInfo, newOrphanedReplicaDataStores map[string]string) error {
path: controller/node_controller.go
user: login: derekbit
body: I think we should use `createOrphans` and `createOrphan` that are similar to `createVolume` and so on. It is a convention and developers already know it is for resource creation.
created_at: 2025-04-21T01:34:41Z
author_association: MEMBER
diff_hunk: @@ -896,6 +900,32 @@ func (sc *SettingController) syncDefaultLonghornStaticStorageClass() error {
 	return err
 }
 
+func (sc *SettingController) syncOrphanResourceAutoDeletion() error {
+	setting, err := sc.ds.GetSettingOrphanResourceAutoDeletion()
+	if err != nil {
+		return err
+	}
+	var replicaDataCleanupValue string
+	if setting[types.OrphanResourceTypeReplicaData] {
+		replicaDataCleanupValue = "true"
+	} else {
+		replicaDataCleanupValue = "false"
+	}
+
+	deprecatedSetting, err := sc.ds.GetSetting(types.SettingNameOrphanAutoDeletion)
+	if err != nil {
+		return err
+	}
+	if replicaDataCleanupValue != deprecatedSetting.Value {
+		deprecatedSetting.Value = replicaDataCleanupValue
+		if _, err := sc.ds.UpdateSetting(deprecatedSetting); err != nil {
+			return err
+		}
+	}
path: controller/setting_controller.go
user: login: COLDTURNIP
body: 1. syncing the new config to deprecated setting would eliminate the user confusion about whether the data store auto cleanup is enabled or not.
2. the new setting will first be synced from old setting in LH manager's upgrade stage, then keep syncing the new setting back to the old one after the setting controller launched.
created_at: 2025-04-21T01:40:31Z
author_association: MEMBER
diff_hunk: @@ -327,22 +508,24 @@ func (oc *OrphanController) deleteOrphanedReplica(orphan *longhorn.Orphan) error
 		err := lhns.DeletePath(filepath.Join(diskPath, "replicas", replicaDirectoryName))
 		return errors.Wrapf(err, "failed to delete orphan replica directory %v in disk %v", replicaDirectoryName, diskPath)
 	case longhorn.DiskTypeBlock:
-		return oc.DeleteV2ReplicaInstance(orphan.Spec.Parameters[longhorn.OrphanDiskName], orphan.Spec.Parameters[longhorn.OrphanDiskUUID], "", orphan.Spec.Parameters[longhorn.OrphanDataName])
+		return oc.DeleteV2ReplicaStore(orphan.Spec.Parameters[longhorn.OrphanDiskName], orphan.Spec.Parameters[longhorn.OrphanDiskUUID], "", orphan.Spec.Parameters[longhorn.OrphanDataName])
path: controller/orphan_controller.go
user: login: derekbit
body: The new name is a bit confusing. Could we still use `DeleteV2ReplicaInstance` because the instance means the SPDK lvol. `Store` sounds weird here.
created_at: 2025-04-21T01:41:00Z
author_association: MEMBER
diff_hunk: @@ -269,48 +303,195 @@ func (oc *OrphanController) isResponsibleFor(orphan *longhorn.Orphan) bool {
 	return isControllerResponsibleFor(oc.controllerID, oc.ds, orphan.Name, orphan.Spec.NodeID, orphan.Status.OwnerID)
 }
 
-func (oc *OrphanController) cleanupOrphanedData(orphan *longhorn.Orphan) (err error) {
+func (oc *OrphanController) cleanupOrphanedResource(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
path: controller/orphan_controller.go
user: login: derekbit
body: I'm fine with the change. 
Resource can mean replica directory, replica instance, engine instance in the data plane.
created_at: 2025-04-21T01:42:20Z
author_association: MEMBER
diff_hunk: @@ -5,12 +5,15 @@ import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 type OrphanType string
 
 const (
-	OrphanTypeReplica = OrphanType("replica")
+	OrphanTypeReplicaDataStore = OrphanType("replica")
path: k8s/pkg/apis/longhorn/v1beta2/orphan.go
user: login: derekbit
body: If we are unable to find a better name, we can continue using `OrphanTypeReplicaData`.
`DataStore` is already used for the dataStore in the control plane. The naming is quite confusing. 
```suggestion
	OrphanTypeReplicaDataStore = OrphanType("replica")
```
created_at: 2025-04-21T01:45:27Z
author_association: MEMBER
diff_hunk: @@ -55,12 +55,13 @@ type InstanceProcessSpec struct {
 type InstanceState string
 
 const (
-	InstanceStateRunning  = InstanceState("running")
-	InstanceStateStopped  = InstanceState("stopped")
-	InstanceStateError    = InstanceState("error")
-	InstanceStateStarting = InstanceState("starting")
-	InstanceStateStopping = InstanceState("stopping")
-	InstanceStateUnknown  = InstanceState("unknown")
+	InstanceStateRunning    = InstanceState("running")
+	InstanceStateStopped    = InstanceState("stopped")
+	InstanceStateError      = InstanceState("error")
+	InstanceStateStarting   = InstanceState("starting")
+	InstanceStateStopping   = InstanceState("stopping")
+	InstanceStateUnknown    = InstanceState("unknown")
+	InstanceStateTerminated = InstanceState("terminated")
path: k8s/pkg/apis/longhorn/v1beta2/instancemanager.go
user: login: COLDTURNIP
body: The `terminated` state represents that an instance was detected by the instance manager, but disappeared from the instance manager later. This is the trigger for orphan controller get notice that an orphan complete the cleanup. While we're going to move the status updating logic to orphan controller, will still keep this for state tracking.
created_at: 2025-04-21T02:48:12Z
author_association: MEMBER
diff_hunk: @@ -20,6 +23,9 @@ const (
 )
 
 const (
+	OrphanInstanceName   = "InstanceName"
+	OrphanDataEngineType = "DataEngineType"
path: k8s/pkg/apis/longhorn/v1beta2/orphan.go
user: login: COLDTURNIP
body: Add new field `DataEngine` in `OrphanSpec`, and the webhook validate when the orphan type is instance orphans.
created_at: 2025-04-21T10:49:55Z
author_association: MEMBER
diff_hunk: @@ -55,12 +55,13 @@ type InstanceProcessSpec struct {
 type InstanceState string
 
 const (
-	InstanceStateRunning  = InstanceState("running")
-	InstanceStateStopped  = InstanceState("stopped")
-	InstanceStateError    = InstanceState("error")
-	InstanceStateStarting = InstanceState("starting")
-	InstanceStateStopping = InstanceState("stopping")
-	InstanceStateUnknown  = InstanceState("unknown")
+	InstanceStateRunning    = InstanceState("running")
+	InstanceStateStopped    = InstanceState("stopped")
+	InstanceStateError      = InstanceState("error")
+	InstanceStateStarting   = InstanceState("starting")
+	InstanceStateStopping   = InstanceState("stopping")
+	InstanceStateUnknown    = InstanceState("unknown")
+	InstanceStateTerminated = InstanceState("terminated")
path: k8s/pkg/apis/longhorn/v1beta2/instancemanager.go
user: login: PhanLe1010
body: I see. Thank you
created_at: 2025-04-22T01:45:33Z
author_association: NONE
diff_hunk: @@ -2044,6 +2087,265 @@ func (m *InstanceManagerMonitor) StopMonitorWithLock() {
 	m.done = true
 }
 
+func (m *InstanceManagerMonitor) syncOrphans(im *longhorn.InstanceManager, instanceProcess map[string]longhorn.InstanceProcess) {
+	engineProcess := map[string]longhorn.InstanceProcess{}
+	replicaProcess := map[string]longhorn.InstanceProcess{}
+	for name, process := range instanceProcess {
+		switch process.Status.Type {
+		case longhorn.InstanceTypeEngine:
+			engineProcess[name] = process
+		case longhorn.InstanceTypeReplica:
+			replicaProcess[name] = process
+		}
+	}
+
+	existOrphans, err := m.getNodeRuntimeOrphanMap()
+	if err != nil {
+		m.logger.WithError(err).Errorf("Failed to list orphans on node %s", m.controllerID)
+		return
+	}
+
+	// create orphan CR for orphaned instances
+	for _, engineInstance := range engineProcess {
+		if isOrphaned, err := m.isEngineOrphaned(engineInstance.Spec.Name); err != nil {
+			m.logger.WithError(err).Errorf("Failed to check orphan for engine instance %v", engineInstance.Spec.Name)
+		} else if isOrphaned {
+			m.logger.Infof("Engine instance %v is orphaned", engineInstance.Spec.Name)
+			err := m.createOrphan(im, engineInstance.Spec.Name, longhorn.OrphanTypeEngineInstance, engineInstance.Spec.DataEngine)
+			if err != nil {
+				m.logger.WithError(err).Errorf("Failed to create orphan for engine instance %v", engineInstance.Spec.Name)
+			}
+		}
+	}
+	for _, replicaInstance := range replicaProcess {
+		if isOrphaned, err := m.isReplicaOrphaned(replicaInstance.Spec.Name); err != nil {
+			m.logger.WithError(err).Errorf("Failed to check orphan for replica instance %v", replicaInstance.Spec.Name)
+		} else if isOrphaned {
+			m.logger.Infof("Replica instance %v is orphaned", replicaInstance.Spec.Name)
+			err := m.createOrphan(im, replicaInstance.Spec.Name, longhorn.OrphanTypeReplicaInstance, replicaInstance.Spec.DataEngine)
+			if err != nil {
+				m.logger.WithError(err).Errorf("Failed to create orphan for replica instance %v", replicaInstance.Spec.Name)
+			}
+		}
+	}
+
+	// update instance state on orphan CRs
path: controller/instance_manager_controller.go
user: login: PhanLe1010
body: Looks like this commend is no longer needed since we don't update the orphan status inside IM controller now?
created_at: 2025-04-22T01:52:21Z
author_association: NONE
diff_hunk: @@ -988,6 +999,15 @@ func (imc *InstanceManagerController) deleteInstanceManagerPDB(im *longhorn.Inst
 	return nil
 }
 
+func (imc *InstanceManagerController) syncOrphanedInstance(im *longhorn.InstanceManager) error {
+	if im.Spec.NodeID != imc.controllerID {
+		// The orphan owner node is no longer able to monitor instance status.
+		// Clear the orphan CRs, and they will be resync after the owner instance manager get back.
+		return imc.cleanupInstanceManagerOrphanInstances(im)
+	}
path: controller/instance_manager_controller.go
user: login: PhanLe1010
body: Just a small note. We may need to be careful with this logic to avoid triggering actual data deletion when the owner of the IM CR temporarily move to new node 
created_at: 2025-04-22T01:59:12Z
author_association: NONE
diff_hunk: @@ -5,12 +5,15 @@ import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 type OrphanType string
 
 const (
-	OrphanTypeReplica = OrphanType("replica")
+	OrphanTypeReplicaData     = OrphanType("replica")
+	OrphanTypeEngineInstance  = OrphanType("engine-instance")
+	OrphanTypeReplicaInstance = OrphanType("replica-instance")
 )
 
 const (
 	OrphanConditionTypeError         = "Error"
 	OrphanConditionTypeDataCleanable = "DataCleanable"
+	OrphanConditionTypeInstanceState = "InstanceState"
path: k8s/pkg/apis/longhorn/v1beta2/orphan.go
user: login: PhanLe1010
body: Look like this condition represent InstanceStateTerminated. Should we change the name? 
created_at: 2025-04-22T02:08:42Z
author_association: NONE
diff_hunk: @@ -5,12 +5,15 @@ import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 type OrphanType string
 
 const (
-	OrphanTypeReplica = OrphanType("replica")
+	OrphanTypeReplicaData     = OrphanType("replica")
+	OrphanTypeEngineInstance  = OrphanType("engine-instance")
+	OrphanTypeReplicaInstance = OrphanType("replica-instance")
 )
 
 const (
 	OrphanConditionTypeError         = "Error"
 	OrphanConditionTypeDataCleanable = "DataCleanable"
+	OrphanConditionTypeInstanceState = "InstanceState"
path: k8s/pkg/apis/longhorn/v1beta2/orphan.go
user: login: COLDTURNIP
body: Sure, will change to `InstanceExist`, positive if instance manager contains this instance, and record the instance state in reason.
created_at: 2025-04-22T02:40:48Z
author_association: MEMBER
diff_hunk: @@ -85,3 +101,49 @@ func upgradeVolumes(namespace string, lhClient *lhclientset.Clientset, resourceM
 
 	return nil
 }
+
+func updateSettings(namespace string, lhClient *lhclientset.Clientset, kubeClient *clientset.Clientset, resourceMaps map[string]interface{}) (err error) {
+	defer func() {
+		err = errors.Wrapf(err, upgradeLogPrefix+"upgrade settings failed")
+	}()
+
+	settingsMap, err := upgradeutil.ListAndUpdateSettingsInProvidedCache(namespace, lhClient, resourceMaps)
+	if err != nil {
+		if apierrors.IsNotFound(err) {
+			return nil
+		}
+		return errors.Wrapf(err, "failed to list all existing Longhorn settings during the settings upgrade")
+	}
path: upgrade/v18xto190/upgrade.go
user: login: derekbit
body: Why need to ListAndUpdateSettingsInProvidedCache again?
created_at: 2025-04-22T03:52:39Z
author_association: MEMBER
diff_hunk: @@ -85,3 +101,49 @@ func upgradeVolumes(namespace string, lhClient *lhclientset.Clientset, resourceM
 
 	return nil
 }
+
+func updateSettings(namespace string, lhClient *lhclientset.Clientset, kubeClient *clientset.Clientset, resourceMaps map[string]interface{}) (err error) {
+	defer func() {
+		err = errors.Wrapf(err, upgradeLogPrefix+"upgrade settings failed")
+	}()
+
+	settingsMap, err := upgradeutil.ListAndUpdateSettingsInProvidedCache(namespace, lhClient, resourceMaps)
+	if err != nil {
+		if apierrors.IsNotFound(err) {
+			return nil
+		}
+		return errors.Wrapf(err, "failed to list all existing Longhorn settings during the settings upgrade")
+	}
+
+	if err := updateOrphanResourceAutoDeletionSetting(settingsMap); err != nil {
+		return errors.Wrapf(err, "failed to update orphan resource auto-deletion settings")
+	}
+
+	return nil
+}
+
+func updateOrphanResourceAutoDeletionSetting(settingsMap map[string]*v1beta2.Setting) (err error) {
+	oldSetting, exist := settingsMap[string(types.SettingNameOrphanAutoDeletion)]
+	if !exist {
+		return nil
+	}
+	newSetting, exist := settingsMap[string(types.SettingNameOrphanResourceAutoDeletion)]
+	if !exist {
+		newSetting = &v1beta2.Setting{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: string(types.SettingNameOrphanResourceAutoDeletion),
+			},
+		}
+		settingsMap[string(types.SettingNameOrphanResourceAutoDeletion)] = newSetting
+	}
+	resourceTypes, _ := types.UnmarshalOrphanResourceTypes(newSetting.Value)
+	resourceTypes[types.OrphanResourceTypeReplicaData] = oldSetting.Value == "true"
+	enabledResourceType := make([]string, 0, len(resourceTypes))
+	for rt, enabled := range resourceTypes {
+		if enabled {
+			enabledResourceType = append(enabledResourceType, string(rt))
+		}
+	}
+	newSetting.Value = strings.Join(enabledResourceType, ";")
+	return nil
+}
path: upgrade/v18xto190/upgrade.go
user: login: derekbit
body: nit: add a unittest for the function
created_at: 2025-04-22T03:56:46Z
author_association: MEMBER
diff_hunk: @@ -32,6 +36,18 @@ func UpgradeResources(namespace string, lhClient *lhclientset.Clientset, kubeCli
 		return errors.New("resourceMaps cannot be nil")
 	}
 
+	if err := updateCRs(namespace, lhClient, kubeClient, resourceMaps); err != nil {
+		return err
+	}
+
+	if err := updateSettings(namespace, lhClient, kubeClient, resourceMaps); err != nil {
path: upgrade/v18xto190/upgrade.go
user: login: derekbit
body: Naming is confusing.
Settings are CRs. Why do we need a separate function `updateSettings`? Need to rename the function for reflecting its purpose.
created_at: 2025-04-22T03:58:09Z
author_association: MEMBER
diff_hunk: @@ -726,6 +746,10 @@ func GetOrphanChecksumNameForOrphanedDataStore(nodeID, diskName, diskPath, diskU
 	return orphanPrefix + util.GetStringChecksumSHA256(strings.TrimSpace(fmt.Sprintf("%s-%s-%s-%s-%s", nodeID, diskName, diskPath, diskUUID, dataStore)))
 }
 
+func GetOrphanChecksumNameForOrphanedInstance(instanceName, nodeID, dataEngineType string) string {
path: types/types.go
user: login: derekbit
body: nit:
```suggestion
func GetOrphanChecksumNameForOrphanedInstance(instanceName, nodeID, dataEngine string) string {
```
created_at: 2025-04-22T04:00:33Z
author_association: MEMBER
diff_hunk: @@ -1055,14 +1058,31 @@ var (
 	SettingDefinitionOrphanAutoDeletion = SettingDefinition{
 		DisplayName: "Orphan Auto-Deletion",
 		Description: "This setting allows Longhorn to delete the orphan resource and its corresponding orphaned data automatically. \n\n" +
-			"Orphan resources on down or unknown nodes will not be cleaned up automatically. \n\n",
+			"Orphan resources on down or unknown nodes will not be cleaned up automatically. \n\n" +
+			fmt.Sprintf("Deprecated: enable \"%s\" in %s instead. \n\n", OrphanResourceTypeReplicaData, SettingNameOrphanResourceAutoDeletion),
 		Category: SettingCategoryOrphan,
 		Type:     SettingTypeBool,
 		Required: true,
-		ReadOnly: false,
+		ReadOnly: true,
 		Default:  "false",
 	}
 
+	SettingDefinitionOrphanResourceAutoDeletion = SettingDefinition{
+		DisplayName: "Orphan Resource Auto-Deletion",
+		Description: "This setting allows Longhorn to delete the orphan resource and its corresponding orphaned resource automatically. \n\n" +
+			"Orphan resources on down or unknown nodes will not be cleaned up automatically. \n\n" +
+			"List the enabled resource types in a semicolon-separate list. \n\n" +
+			"Available items are: \n\n" +
+			"- **replicaData**: replica data store \n\n" +
+			"- **engineInstance**: engine runtime instance \n\n" +
+			"- **replicaInstance**: replica runtime instance \n\n",
path: types/setting.go
user: login: derekbit
body: @sushant-suse Could you help review the description? Thanks.
created_at: 2025-04-22T04:01:42Z
author_association: MEMBER
diff_hunk: @@ -1771,6 +1799,32 @@ func UnmarshalNodeSelector(nodeSelectorSetting string) (map[string]string, error
 	return nodeSelector, nil
 }
 
+func UnmarshalOrphanResourceTypes(resourceTypesSetting string) (map[OrphanResourceType]bool, error) {
path: types/setting.go
user: login: derekbit
body: Add a unittest for the function.
created_at: 2025-04-22T04:02:14Z
author_association: MEMBER
diff_hunk: @@ -1610,6 +1630,22 @@ func (imc *InstanceManagerController) createInstanceManagerPodSpec(im *longhorn.
 	return podSpec, nil
 }
 
+func (imc *InstanceManagerController) cleanupInstanceManagerOrphanInstances(im *longhorn.InstanceManager) error {
+	orphans, err := imc.ds.ListOrphansByNodeRO(im.Spec.NodeID)
+	if err != nil {
+		return err
+	}
+	for _, orphan := range orphans {
+		switch orphan.Spec.Type {
+		case longhorn.OrphanTypeEngineInstance, longhorn.OrphanTypeReplicaInstance:
+			if err := imc.ds.DeleteOrphan(orphan.Name); err != nil && !apierrors.IsNotFound(err) {
+				return err
+			}
path: controller/instance_manager_controller.go
user: login: derekbit
body: Use `multierr` to record the err and continue the next deletions.
created_at: 2025-04-22T04:03:50Z
author_association: MEMBER
diff_hunk: @@ -227,13 +263,11 @@ func (oc *OrphanController) reconcile(orphanName string) (err error) {
 	}
 
 	if !orphan.DeletionTimestamp.IsZero() {
-		defer func() {
-			if err == nil {
-				err = oc.ds.RemoveFinalizerForOrphan(orphan)
-			}
-		}()
-
-		return oc.cleanupOrphanedData(orphan)
+		isCleanupComplete, err := oc.cleanupOrphanedResource(orphan)
path: controller/orphan_controller.go
user: login: derekbit
body: Is it possible `isCleanupComplete` is `false` but `err` is `nil`?
created_at: 2025-04-22T04:24:06Z
author_association: MEMBER
diff_hunk: @@ -269,48 +303,200 @@ func (oc *OrphanController) isResponsibleFor(orphan *longhorn.Orphan) bool {
 	return isControllerResponsibleFor(oc.controllerID, oc.ds, orphan.Name, orphan.Spec.NodeID, orphan.Status.OwnerID)
 }
 
-func (oc *OrphanController) cleanupOrphanedData(orphan *longhorn.Orphan) (err error) {
+func (oc *OrphanController) cleanupOrphanedResource(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
 	log := getLoggerForOrphan(oc.logger, orphan)
 
 	defer func() {
 		if err == nil {
 			return
 		}
 
-		err = errors.Wrapf(err, "failed to delete orphan %v data", orphan.Name)
+		err = errors.Wrapf(err, "failed to delete %v orphan %v", orphan.Spec.Type, orphan.Name)
 		orphan.Status.Conditions = types.SetCondition(orphan.Status.Conditions,
 			longhorn.OrphanConditionTypeError, longhorn.ConditionStatusTrue, "", err.Error())
 	}()
 
 	// Make sure if the orphan nodeID and controller ID are same.
 	// If NO, just delete the orphan resource object and don't touch the data.
 	if orphan.Spec.NodeID != oc.controllerID {
-		log.Infof("Orphan nodeID %v is different from controllerID %v, so just delete the orphan resource object",
-			orphan.Name, oc.controllerID)
-		return nil
+		log.WithFields(logrus.Fields{
+			"orphanType": orphan.Spec.Type,
+			"orphanName": orphan.Name,
+			"orphanNode": orphan.Spec.NodeID,
+		}).Infof("Orphan does not belong to this controller. Skipping resource cleanup")
+		return true, nil
 	}
 
-	if types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeDataCleanable).Status !=
-		longhorn.ConditionStatusTrue {
-		log.Infof("Only delete orphan %v resource object and do not delete the orphaned data", orphan.Name)
-		return nil
+	switch orphan.Spec.Type {
+	case longhorn.OrphanTypeEngineInstance:
+		isCleanupComplete, err = oc.cleanupOrphanedEngineInstance(orphan)
+	case longhorn.OrphanTypeReplicaInstance:
+		isCleanupComplete, err = oc.cleanupOrphanedReplicaInstance(orphan)
+	case longhorn.OrphanTypeReplicaData:
+		if types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeDataCleanable).Status !=
+			longhorn.ConditionStatusTrue {
+			log.Infof("Only delete orphan %v resource object and do not delete the orphaned data store", orphan.Name)
+			return true, nil
+		}
+		err = oc.deleteOrphanedReplicaDataStore(orphan)
+		if err == nil || (err != nil && apierrors.IsNotFound(err)) {
+			isCleanupComplete = true
+			err = nil
+		}
+	default:
+		return false, fmt.Errorf("unknown orphan type %v to clear orphan %v", orphan.Spec.Type, orphan.Name)
path: controller/orphan_controller.go
user: login: derekbit
body: 
```suggestion
		return false, fmt.Errorf("unknown orphan type %v to clean up orphaned resource for %v", orphan.Spec.Type, orphan.Name)
```
created_at: 2025-04-22T04:27:13Z
author_association: MEMBER
diff_hunk: @@ -334,15 +520,17 @@ func (oc *OrphanController) deleteOrphanedReplica(orphan *longhorn.Orphan) error
 }
 
 func (oc *OrphanController) DeleteV2ReplicaInstance(diskName, diskUUID, diskDriver, replicaInstanceName string) (err error) {
-	logrus.Infof("Deleting SPDK replica instance %v on disk %v on node %v", replicaInstanceName, diskUUID, oc.controllerID)
+	logrus.Infof("Deleting SPDK replica store %v on disk %v on node %v", replicaInstanceName, diskUUID, oc.controllerID)
 
 	defer func() {
-		err = errors.Wrapf(err, "cannot delete v2 replica instance %v", replicaInstanceName)
+		if err != nil {
+			err = errors.Wrapf(err, "cannot delete v2 replica store %v", replicaInstanceName)
path: controller/orphan_controller.go
user: login: derekbit
body: 
```suggestion
			err = errors.Wrapf(err, "cannot delete v2 replica instance %v", replicaInstanceName)
```
created_at: 2025-04-22T04:28:33Z
author_association: MEMBER
diff_hunk: @@ -334,15 +520,17 @@ func (oc *OrphanController) deleteOrphanedReplica(orphan *longhorn.Orphan) error
 }
 
 func (oc *OrphanController) DeleteV2ReplicaInstance(diskName, diskUUID, diskDriver, replicaInstanceName string) (err error) {
-	logrus.Infof("Deleting SPDK replica instance %v on disk %v on node %v", replicaInstanceName, diskUUID, oc.controllerID)
+	logrus.Infof("Deleting SPDK replica store %v on disk %v on node %v", replicaInstanceName, diskUUID, oc.controllerID)
path: controller/orphan_controller.go
user: login: derekbit
body: 
```suggestion
	logrus.Infof("Deleting SPDK replica instance %v on disk %v on node %v", replicaInstanceName, diskUUID, oc.controllerID)
```
created_at: 2025-04-22T04:28:39Z
author_association: MEMBER
diff_hunk: @@ -334,15 +520,17 @@ func (oc *OrphanController) deleteOrphanedReplica(orphan *longhorn.Orphan) error
 }
 
 func (oc *OrphanController) DeleteV2ReplicaInstance(diskName, diskUUID, diskDriver, replicaInstanceName string) (err error) {
-	logrus.Infof("Deleting SPDK replica instance %v on disk %v on node %v", replicaInstanceName, diskUUID, oc.controllerID)
+	logrus.Infof("Deleting SPDK replica store %v on disk %v on node %v", replicaInstanceName, diskUUID, oc.controllerID)
 
 	defer func() {
-		err = errors.Wrapf(err, "cannot delete v2 replica instance %v", replicaInstanceName)
+		if err != nil {
+			err = errors.Wrapf(err, "cannot delete v2 replica store %v", replicaInstanceName)
+		}
 	}()
 
 	im, err := oc.ds.GetRunningInstanceManagerByNodeRO(oc.controllerID, longhorn.DataEngineTypeV2)
 	if err != nil {
-		return errors.Wrapf(err, "failed to get running instance manager for node %v for deleting v2 replica instance %v", oc.controllerID, replicaInstanceName)
+		return errors.Wrapf(err, "failed to get running instance manager for node %v for deleting v2 replica store %v", oc.controllerID, replicaInstanceName)
path: controller/orphan_controller.go
user: login: derekbit
body: 
```suggestion
		return errors.Wrapf(err, "failed to get running instance manager for node %v for deleting v2 replica instance %v", oc.controllerID, replicaInstanceName)
```
created_at: 2025-04-22T04:28:49Z
author_association: MEMBER
diff_hunk: @@ -371,6 +572,55 @@ func (oc *OrphanController) updateConditions(orphan *longhorn.Orphan) error {
 	return nil
 }
 
+// If the InstanceState condition status is true, the corresponding instance is still exist in instance manager
+func (oc *OrphanController) updateInstanceStateCondition(orphan *longhorn.Orphan, instanceType longhorn.InstanceType) (err error) {
+	var instanceState longhorn.InstanceState
+	defer func() {
+		if err != nil {
+			return
+		}
+
+		status := longhorn.ConditionStatusTrue
+		if instanceState == longhorn.InstanceStateTerminated {
+			status = longhorn.ConditionStatusFalse
+		}
+
+		orphan.Status.Conditions = types.SetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeInstanceState, status, string(instanceState), "")
+	}()
+
+	instanceName, instanceManager, err := oc.extractOrphanedInstanceInfo(orphan)
+	im, err := oc.ds.GetInstanceManager(instanceManager)
+	if err != nil {
+		if !apierrors.IsNotFound(err) {
+			oc.logger.WithError(err).Infof("No instance manager for node %v for update instance state of orphan instance %v", oc.controllerID, orphan.Name)
+			instanceState = longhorn.InstanceStateTerminated
+			return nil
+		}
+		return err
+	}
+	imc, err := engineapi.NewInstanceManagerClient(im, false)
+	if err != nil {
+		return errors.Wrapf(err, "failed to check node down or missing manager for node %v", orphan.Spec.NodeID)
+	}
+	defer func() {
+		if closeErr := imc.Close(); closeErr != nil {
+			oc.logger.WithError(closeErr).Error("failed to close instance manager client")
+		}
+	}()
+
+	instanceProc, err := imc.InstanceGet(orphan.Spec.DataEngine, instanceName, string(instanceType))
+	switch {
+	case err != nil:
+		return errors.Wrapf(err, "failed to get instance %v", instanceName)
+	case instanceProc == nil:
+		instanceState = longhorn.InstanceStateTerminated
+	default:
+		instanceState = instanceProc.Status.State
+	}
path: controller/orphan_controller.go
user: login: derekbit
body: What's the purpose of reading the instance state?
created_at: 2025-04-22T04:30:40Z
author_association: MEMBER
diff_hunk: @@ -612,6 +617,12 @@ func (imc *InstanceManagerController) handlePod(im *longhorn.InstanceManager) er
 		return err
 	}
 
+	if isPodDeletedOrNotRunning {
+		if err := imc.cleanupInstanceManagerOrphanInstances(im); err != nil {
path: controller/instance_manager_controller.go
user: login: derekbit
body: The function `cleanupInstanceManagerOrphanInstances` actually deletes Orphan resources rather than instances. Suggest renaming to `deleteOrphans(im)`.

```suggestion
		if err := imc.deleteOrphans(im); err != nil {
```
created_at: 2025-04-22T07:39:32Z
author_association: MEMBER
diff_hunk: @@ -612,6 +617,12 @@ func (imc *InstanceManagerController) handlePod(im *longhorn.InstanceManager) er
 		return err
 	}
 
+	if isPodDeletedOrNotRunning {
+		if err := imc.cleanupInstanceManagerOrphanInstances(im); err != nil {
+			log.WithError(err).Warnf("Instance manager %v failed to cleanup orphan instance for stopped instance manager pod", im.Name)
path: controller/instance_manager_controller.go
user: login: derekbit
body: Update the log message.
created_at: 2025-04-22T07:39:51Z
author_association: MEMBER
diff_hunk: @@ -2044,6 +2087,265 @@ func (m *InstanceManagerMonitor) StopMonitorWithLock() {
 	m.done = true
 }
 
+func (m *InstanceManagerMonitor) syncOrphans(im *longhorn.InstanceManager, instanceProcess map[string]longhorn.InstanceProcess) {
+	engineProcess := map[string]longhorn.InstanceProcess{}
+	replicaProcess := map[string]longhorn.InstanceProcess{}
+	for name, process := range instanceProcess {
+		switch process.Status.Type {
+		case longhorn.InstanceTypeEngine:
+			engineProcess[name] = process
+		case longhorn.InstanceTypeReplica:
+			replicaProcess[name] = process
+		}
+	}
path: controller/instance_manager_controller.go
user: login: derekbit
body: nit: wrap the codes into a helper function like `categorizeProcesses()`
created_at: 2025-04-22T07:43:50Z
author_association: MEMBER
diff_hunk: @@ -2044,6 +2087,265 @@ func (m *InstanceManagerMonitor) StopMonitorWithLock() {
 	m.done = true
 }
 
+func (m *InstanceManagerMonitor) syncOrphans(im *longhorn.InstanceManager, instanceProcess map[string]longhorn.InstanceProcess) {
+	engineProcess := map[string]longhorn.InstanceProcess{}
+	replicaProcess := map[string]longhorn.InstanceProcess{}
+	for name, process := range instanceProcess {
+		switch process.Status.Type {
+		case longhorn.InstanceTypeEngine:
+			engineProcess[name] = process
+		case longhorn.InstanceTypeReplica:
+			replicaProcess[name] = process
+		}
+	}
+
+	existOrphans, err := m.getNodeRuntimeOrphanMap()
+	if err != nil {
+		m.logger.WithError(err).Errorf("Failed to list orphans on node %s", m.controllerID)
+		return
+	}
+
+	// create orphan CR for orphaned instances
+	for _, engineInstance := range engineProcess {
+		if isOrphaned, err := m.isEngineOrphaned(engineInstance.Spec.Name); err != nil {
+			m.logger.WithError(err).Errorf("Failed to check orphan for engine instance %v", engineInstance.Spec.Name)
+		} else if isOrphaned {
+			m.logger.Infof("Engine instance %v is orphaned", engineInstance.Spec.Name)
+			err := m.createOrphan(im, engineInstance.Spec.Name, longhorn.OrphanTypeEngineInstance, engineInstance.Spec.DataEngine)
+			if err != nil {
+				m.logger.WithError(err).Errorf("Failed to create orphan for engine instance %v", engineInstance.Spec.Name)
+			}
+		}
+	}
+	for _, replicaInstance := range replicaProcess {
+		if isOrphaned, err := m.isReplicaOrphaned(replicaInstance.Spec.Name); err != nil {
+			m.logger.WithError(err).Errorf("Failed to check orphan for replica instance %v", replicaInstance.Spec.Name)
+		} else if isOrphaned {
+			m.logger.Infof("Replica instance %v is orphaned", replicaInstance.Spec.Name)
+			err := m.createOrphan(im, replicaInstance.Spec.Name, longhorn.OrphanTypeReplicaInstance, replicaInstance.Spec.DataEngine)
+			if err != nil {
+				m.logger.WithError(err).Errorf("Failed to create orphan for replica instance %v", replicaInstance.Spec.Name)
+			}
+		}
+	}
+
+	// update instance state on orphan CRs
+	deleteOrphans := make(map[string]bool)
+	for orphanName, orphan := range existOrphans {
+		instanceName := orphan.Spec.Parameters[longhorn.OrphanInstanceName]
+		var instanceProc longhorn.InstanceProcess
+		var instanceExist, instanceCRScheduledBack bool
+		switch orphan.Spec.Type {
+		case longhorn.OrphanTypeEngineInstance:
+			instanceProc, instanceExist = engineProcess[instanceName]
+			isCRScheduledBack, err := m.isEngineScheduledOnNode(instanceName)
+			if err != nil {
+				m.logger.WithError(err).WithField("instance", instanceName).Errorf("Failed to check CR scheduled node for engine instance orphan %v", orphanName)
+				instanceCRScheduledBack = false
+			} else {
+				instanceCRScheduledBack = isCRScheduledBack
+			}
+		case longhorn.OrphanTypeReplicaInstance:
+			instanceProc, instanceExist = replicaProcess[instanceName]
+			isCRScheduledBack, err := m.isReplicaScheduledOnNode(instanceName)
+			if err != nil {
+				m.logger.WithError(err).WithField("instance", instanceName).Errorf("Failed to check CR scheduled node for replica instance orphan %v", orphanName)
+				instanceCRScheduledBack = false
+			} else {
+				instanceCRScheduledBack = isCRScheduledBack
+			}
+		}
+		var instanceState longhorn.InstanceState
+		if instanceExist {
+			instanceState = instanceProc.Status.State
+		} else {
+			instanceState = longhorn.InstanceStateTerminated
+		}
+
+		if orphan.DeletionTimestamp.IsZero() {
+			if !instanceExist || instanceCRScheduledBack {
+				m.logger.WithFields(logrus.Fields{
+					"instanceExist":           instanceExist,
+					"instanceState":           instanceState,
+					"instanceCRScheduledBack": instanceCRScheduledBack,
+				}).Debugf("Orphan instance %s is deletable", orphanName)
+				deleteOrphans[orphanName] = true
+			}
+		}
+	}
+
+	// delete orphan CRs
+	autoDeletionTypes, err := m.ds.GetSettingOrphanResourceAutoDeletion()
+	var autoDeleteEngine = false
+	var autoDeleteReplica = false
+	if err != nil {
+		m.logger.WithError(err).Warnf("Failed to fetch orphan auto deletion setting, disable by default")
+	} else {
+		autoDeleteEngine = autoDeletionTypes[types.OrphanResourceTypeEngineInstance]
+		autoDeleteReplica = autoDeletionTypes[types.OrphanResourceTypeReplicaInstance]
+	}
+	m.cleanupOrphans(existOrphans, deleteOrphans, autoDeleteEngine, autoDeleteReplica)
+}
+
+func (m *InstanceManagerMonitor) getNodeRuntimeOrphanMap() (map[string]*longhorn.Orphan, error) {
path: controller/instance_manager_controller.go
user: login: derekbit
body: It is not easy to quickly understand the purpose of the function from the function name.
Suggest `listOrphansForEngineAndReplicaInstances(nodeName string)` or similar one.
created_at: 2025-04-22T07:47:29Z
author_association: MEMBER
diff_hunk: @@ -2044,6 +2087,265 @@ func (m *InstanceManagerMonitor) StopMonitorWithLock() {
 	m.done = true
 }
 
+func (m *InstanceManagerMonitor) syncOrphans(im *longhorn.InstanceManager, instanceProcess map[string]longhorn.InstanceProcess) {
+	engineProcess := map[string]longhorn.InstanceProcess{}
+	replicaProcess := map[string]longhorn.InstanceProcess{}
+	for name, process := range instanceProcess {
+		switch process.Status.Type {
+		case longhorn.InstanceTypeEngine:
+			engineProcess[name] = process
+		case longhorn.InstanceTypeReplica:
+			replicaProcess[name] = process
+		}
+	}
+
+	existOrphans, err := m.getNodeRuntimeOrphanMap()
+	if err != nil {
+		m.logger.WithError(err).Errorf("Failed to list orphans on node %s", m.controllerID)
+		return
+	}
+
+	// create orphan CR for orphaned instances
+	for _, engineInstance := range engineProcess {
+		if isOrphaned, err := m.isEngineOrphaned(engineInstance.Spec.Name); err != nil {
+			m.logger.WithError(err).Errorf("Failed to check orphan for engine instance %v", engineInstance.Spec.Name)
+		} else if isOrphaned {
+			m.logger.Infof("Engine instance %v is orphaned", engineInstance.Spec.Name)
+			err := m.createOrphan(im, engineInstance.Spec.Name, longhorn.OrphanTypeEngineInstance, engineInstance.Spec.DataEngine)
+			if err != nil {
+				m.logger.WithError(err).Errorf("Failed to create orphan for engine instance %v", engineInstance.Spec.Name)
+			}
+		}
+	}
+	for _, replicaInstance := range replicaProcess {
+		if isOrphaned, err := m.isReplicaOrphaned(replicaInstance.Spec.Name); err != nil {
+			m.logger.WithError(err).Errorf("Failed to check orphan for replica instance %v", replicaInstance.Spec.Name)
+		} else if isOrphaned {
+			m.logger.Infof("Replica instance %v is orphaned", replicaInstance.Spec.Name)
+			err := m.createOrphan(im, replicaInstance.Spec.Name, longhorn.OrphanTypeReplicaInstance, replicaInstance.Spec.DataEngine)
+			if err != nil {
+				m.logger.WithError(err).Errorf("Failed to create orphan for replica instance %v", replicaInstance.Spec.Name)
+			}
+		}
+	}
path: controller/instance_manager_controller.go
user: login: derekbit
body: Wrap the codes into a helper function like `createOrphansForInstances`
created_at: 2025-04-22T07:48:58Z
author_association: MEMBER
diff_hunk: @@ -2044,6 +2087,265 @@ func (m *InstanceManagerMonitor) StopMonitorWithLock() {
 	m.done = true
 }
 
+func (m *InstanceManagerMonitor) syncOrphans(im *longhorn.InstanceManager, instanceProcess map[string]longhorn.InstanceProcess) {
+	engineProcess := map[string]longhorn.InstanceProcess{}
+	replicaProcess := map[string]longhorn.InstanceProcess{}
+	for name, process := range instanceProcess {
+		switch process.Status.Type {
+		case longhorn.InstanceTypeEngine:
+			engineProcess[name] = process
+		case longhorn.InstanceTypeReplica:
+			replicaProcess[name] = process
+		}
+	}
+
+	existOrphans, err := m.getNodeRuntimeOrphanMap()
+	if err != nil {
+		m.logger.WithError(err).Errorf("Failed to list orphans on node %s", m.controllerID)
+		return
+	}
+
+	// create orphan CR for orphaned instances
+	for _, engineInstance := range engineProcess {
+		if isOrphaned, err := m.isEngineOrphaned(engineInstance.Spec.Name); err != nil {
+			m.logger.WithError(err).Errorf("Failed to check orphan for engine instance %v", engineInstance.Spec.Name)
+		} else if isOrphaned {
+			m.logger.Infof("Engine instance %v is orphaned", engineInstance.Spec.Name)
+			err := m.createOrphan(im, engineInstance.Spec.Name, longhorn.OrphanTypeEngineInstance, engineInstance.Spec.DataEngine)
+			if err != nil {
+				m.logger.WithError(err).Errorf("Failed to create orphan for engine instance %v", engineInstance.Spec.Name)
+			}
+		}
+	}
+	for _, replicaInstance := range replicaProcess {
+		if isOrphaned, err := m.isReplicaOrphaned(replicaInstance.Spec.Name); err != nil {
+			m.logger.WithError(err).Errorf("Failed to check orphan for replica instance %v", replicaInstance.Spec.Name)
+		} else if isOrphaned {
+			m.logger.Infof("Replica instance %v is orphaned", replicaInstance.Spec.Name)
+			err := m.createOrphan(im, replicaInstance.Spec.Name, longhorn.OrphanTypeReplicaInstance, replicaInstance.Spec.DataEngine)
+			if err != nil {
+				m.logger.WithError(err).Errorf("Failed to create orphan for replica instance %v", replicaInstance.Spec.Name)
+			}
+		}
+	}
+
+	// update instance state on orphan CRs
+	deleteOrphans := make(map[string]bool)
path: controller/instance_manager_controller.go
user: login: derekbit
body: 
```suggestion
	deletableOrphans := make(map[string]bool)
```
created_at: 2025-04-22T07:49:24Z
author_association: MEMBER
diff_hunk: @@ -2044,6 +2087,265 @@ func (m *InstanceManagerMonitor) StopMonitorWithLock() {
 	m.done = true
 }
 
+func (m *InstanceManagerMonitor) syncOrphans(im *longhorn.InstanceManager, instanceProcess map[string]longhorn.InstanceProcess) {
+	engineProcess := map[string]longhorn.InstanceProcess{}
+	replicaProcess := map[string]longhorn.InstanceProcess{}
+	for name, process := range instanceProcess {
+		switch process.Status.Type {
+		case longhorn.InstanceTypeEngine:
+			engineProcess[name] = process
+		case longhorn.InstanceTypeReplica:
+			replicaProcess[name] = process
+		}
+	}
+
+	existOrphans, err := m.getNodeRuntimeOrphanMap()
+	if err != nil {
+		m.logger.WithError(err).Errorf("Failed to list orphans on node %s", m.controllerID)
+		return
+	}
+
+	// create orphan CR for orphaned instances
+	for _, engineInstance := range engineProcess {
+		if isOrphaned, err := m.isEngineOrphaned(engineInstance.Spec.Name); err != nil {
+			m.logger.WithError(err).Errorf("Failed to check orphan for engine instance %v", engineInstance.Spec.Name)
+		} else if isOrphaned {
+			m.logger.Infof("Engine instance %v is orphaned", engineInstance.Spec.Name)
+			err := m.createOrphan(im, engineInstance.Spec.Name, longhorn.OrphanTypeEngineInstance, engineInstance.Spec.DataEngine)
+			if err != nil {
+				m.logger.WithError(err).Errorf("Failed to create orphan for engine instance %v", engineInstance.Spec.Name)
+			}
+		}
+	}
+	for _, replicaInstance := range replicaProcess {
+		if isOrphaned, err := m.isReplicaOrphaned(replicaInstance.Spec.Name); err != nil {
+			m.logger.WithError(err).Errorf("Failed to check orphan for replica instance %v", replicaInstance.Spec.Name)
+		} else if isOrphaned {
+			m.logger.Infof("Replica instance %v is orphaned", replicaInstance.Spec.Name)
+			err := m.createOrphan(im, replicaInstance.Spec.Name, longhorn.OrphanTypeReplicaInstance, replicaInstance.Spec.DataEngine)
+			if err != nil {
+				m.logger.WithError(err).Errorf("Failed to create orphan for replica instance %v", replicaInstance.Spec.Name)
+			}
+		}
+	}
+
+	// update instance state on orphan CRs
+	deleteOrphans := make(map[string]bool)
+	for orphanName, orphan := range existOrphans {
+		instanceName := orphan.Spec.Parameters[longhorn.OrphanInstanceName]
+		var instanceProc longhorn.InstanceProcess
+		var instanceExist, instanceCRScheduledBack bool
+		switch orphan.Spec.Type {
+		case longhorn.OrphanTypeEngineInstance:
+			instanceProc, instanceExist = engineProcess[instanceName]
+			isCRScheduledBack, err := m.isEngineScheduledOnNode(instanceName)
+			if err != nil {
+				m.logger.WithError(err).WithField("instance", instanceName).Errorf("Failed to check CR scheduled node for engine instance orphan %v", orphanName)
+				instanceCRScheduledBack = false
+			} else {
+				instanceCRScheduledBack = isCRScheduledBack
+			}
+		case longhorn.OrphanTypeReplicaInstance:
+			instanceProc, instanceExist = replicaProcess[instanceName]
+			isCRScheduledBack, err := m.isReplicaScheduledOnNode(instanceName)
+			if err != nil {
+				m.logger.WithError(err).WithField("instance", instanceName).Errorf("Failed to check CR scheduled node for replica instance orphan %v", orphanName)
+				instanceCRScheduledBack = false
+			} else {
+				instanceCRScheduledBack = isCRScheduledBack
+			}
+		}
+		var instanceState longhorn.InstanceState
+		if instanceExist {
+			instanceState = instanceProc.Status.State
+		} else {
+			instanceState = longhorn.InstanceStateTerminated
+		}
+
+		if orphan.DeletionTimestamp.IsZero() {
+			if !instanceExist || instanceCRScheduledBack {
+				m.logger.WithFields(logrus.Fields{
+					"instanceExist":           instanceExist,
+					"instanceState":           instanceState,
+					"instanceCRScheduledBack": instanceCRScheduledBack,
+				}).Debugf("Orphan instance %s is deletable", orphanName)
+				deleteOrphans[orphanName] = true
+			}
+		}
+	}
path: controller/instance_manager_controller.go
user: login: derekbit
body: Wrap the codes into a helper function
created_at: 2025-04-22T07:52:05Z
author_association: MEMBER
diff_hunk: @@ -2044,6 +2087,265 @@ func (m *InstanceManagerMonitor) StopMonitorWithLock() {
 	m.done = true
 }
 
+func (m *InstanceManagerMonitor) syncOrphans(im *longhorn.InstanceManager, instanceProcess map[string]longhorn.InstanceProcess) {
+	engineProcess := map[string]longhorn.InstanceProcess{}
+	replicaProcess := map[string]longhorn.InstanceProcess{}
+	for name, process := range instanceProcess {
+		switch process.Status.Type {
+		case longhorn.InstanceTypeEngine:
+			engineProcess[name] = process
+		case longhorn.InstanceTypeReplica:
+			replicaProcess[name] = process
+		}
+	}
+
+	existOrphans, err := m.getNodeRuntimeOrphanMap()
+	if err != nil {
+		m.logger.WithError(err).Errorf("Failed to list orphans on node %s", m.controllerID)
+		return
+	}
+
+	// create orphan CR for orphaned instances
+	for _, engineInstance := range engineProcess {
+		if isOrphaned, err := m.isEngineOrphaned(engineInstance.Spec.Name); err != nil {
+			m.logger.WithError(err).Errorf("Failed to check orphan for engine instance %v", engineInstance.Spec.Name)
+		} else if isOrphaned {
+			m.logger.Infof("Engine instance %v is orphaned", engineInstance.Spec.Name)
+			err := m.createOrphan(im, engineInstance.Spec.Name, longhorn.OrphanTypeEngineInstance, engineInstance.Spec.DataEngine)
+			if err != nil {
+				m.logger.WithError(err).Errorf("Failed to create orphan for engine instance %v", engineInstance.Spec.Name)
+			}
+		}
+	}
+	for _, replicaInstance := range replicaProcess {
+		if isOrphaned, err := m.isReplicaOrphaned(replicaInstance.Spec.Name); err != nil {
+			m.logger.WithError(err).Errorf("Failed to check orphan for replica instance %v", replicaInstance.Spec.Name)
+		} else if isOrphaned {
+			m.logger.Infof("Replica instance %v is orphaned", replicaInstance.Spec.Name)
+			err := m.createOrphan(im, replicaInstance.Spec.Name, longhorn.OrphanTypeReplicaInstance, replicaInstance.Spec.DataEngine)
+			if err != nil {
+				m.logger.WithError(err).Errorf("Failed to create orphan for replica instance %v", replicaInstance.Spec.Name)
+			}
+		}
+	}
+
+	// update instance state on orphan CRs
+	deleteOrphans := make(map[string]bool)
+	for orphanName, orphan := range existOrphans {
+		instanceName := orphan.Spec.Parameters[longhorn.OrphanInstanceName]
+		var instanceProc longhorn.InstanceProcess
+		var instanceExist, instanceCRScheduledBack bool
+		switch orphan.Spec.Type {
+		case longhorn.OrphanTypeEngineInstance:
+			instanceProc, instanceExist = engineProcess[instanceName]
+			isCRScheduledBack, err := m.isEngineScheduledOnNode(instanceName)
+			if err != nil {
+				m.logger.WithError(err).WithField("instance", instanceName).Errorf("Failed to check CR scheduled node for engine instance orphan %v", orphanName)
+				instanceCRScheduledBack = false
+			} else {
+				instanceCRScheduledBack = isCRScheduledBack
+			}
+		case longhorn.OrphanTypeReplicaInstance:
+			instanceProc, instanceExist = replicaProcess[instanceName]
+			isCRScheduledBack, err := m.isReplicaScheduledOnNode(instanceName)
+			if err != nil {
+				m.logger.WithError(err).WithField("instance", instanceName).Errorf("Failed to check CR scheduled node for replica instance orphan %v", orphanName)
+				instanceCRScheduledBack = false
+			} else {
+				instanceCRScheduledBack = isCRScheduledBack
+			}
+		}
+		var instanceState longhorn.InstanceState
+		if instanceExist {
+			instanceState = instanceProc.Status.State
+		} else {
+			instanceState = longhorn.InstanceStateTerminated
+		}
+
+		if orphan.DeletionTimestamp.IsZero() {
+			if !instanceExist || instanceCRScheduledBack {
+				m.logger.WithFields(logrus.Fields{
+					"instanceExist":           instanceExist,
+					"instanceState":           instanceState,
+					"instanceCRScheduledBack": instanceCRScheduledBack,
+				}).Debugf("Orphan instance %s is deletable", orphanName)
+				deleteOrphans[orphanName] = true
+			}
+		}
+	}
+
+	// delete orphan CRs
+	autoDeletionTypes, err := m.ds.GetSettingOrphanResourceAutoDeletion()
+	var autoDeleteEngine = false
+	var autoDeleteReplica = false
+	if err != nil {
+		m.logger.WithError(err).Warnf("Failed to fetch orphan auto deletion setting, disable by default")
path: controller/instance_manager_controller.go
user: login: derekbit
body: should we directly return if run into an error?
created_at: 2025-04-22T07:53:03Z
author_association: MEMBER
diff_hunk: @@ -2044,6 +2087,265 @@ func (m *InstanceManagerMonitor) StopMonitorWithLock() {
 	m.done = true
 }
 
+func (m *InstanceManagerMonitor) syncOrphans(im *longhorn.InstanceManager, instanceProcess map[string]longhorn.InstanceProcess) {
+	engineProcess := map[string]longhorn.InstanceProcess{}
+	replicaProcess := map[string]longhorn.InstanceProcess{}
+	for name, process := range instanceProcess {
+		switch process.Status.Type {
+		case longhorn.InstanceTypeEngine:
+			engineProcess[name] = process
+		case longhorn.InstanceTypeReplica:
+			replicaProcess[name] = process
+		}
+	}
+
+	existOrphans, err := m.getNodeRuntimeOrphanMap()
+	if err != nil {
+		m.logger.WithError(err).Errorf("Failed to list orphans on node %s", m.controllerID)
+		return
+	}
+
+	// create orphan CR for orphaned instances
+	for _, engineInstance := range engineProcess {
+		if isOrphaned, err := m.isEngineOrphaned(engineInstance.Spec.Name); err != nil {
+			m.logger.WithError(err).Errorf("Failed to check orphan for engine instance %v", engineInstance.Spec.Name)
+		} else if isOrphaned {
+			m.logger.Infof("Engine instance %v is orphaned", engineInstance.Spec.Name)
+			err := m.createOrphan(im, engineInstance.Spec.Name, longhorn.OrphanTypeEngineInstance, engineInstance.Spec.DataEngine)
+			if err != nil {
+				m.logger.WithError(err).Errorf("Failed to create orphan for engine instance %v", engineInstance.Spec.Name)
+			}
+		}
+	}
+	for _, replicaInstance := range replicaProcess {
+		if isOrphaned, err := m.isReplicaOrphaned(replicaInstance.Spec.Name); err != nil {
+			m.logger.WithError(err).Errorf("Failed to check orphan for replica instance %v", replicaInstance.Spec.Name)
+		} else if isOrphaned {
+			m.logger.Infof("Replica instance %v is orphaned", replicaInstance.Spec.Name)
+			err := m.createOrphan(im, replicaInstance.Spec.Name, longhorn.OrphanTypeReplicaInstance, replicaInstance.Spec.DataEngine)
+			if err != nil {
+				m.logger.WithError(err).Errorf("Failed to create orphan for replica instance %v", replicaInstance.Spec.Name)
+			}
+		}
+	}
+
+	// update instance state on orphan CRs
+	deleteOrphans := make(map[string]bool)
+	for orphanName, orphan := range existOrphans {
+		instanceName := orphan.Spec.Parameters[longhorn.OrphanInstanceName]
+		var instanceProc longhorn.InstanceProcess
+		var instanceExist, instanceCRScheduledBack bool
+		switch orphan.Spec.Type {
+		case longhorn.OrphanTypeEngineInstance:
+			instanceProc, instanceExist = engineProcess[instanceName]
+			isCRScheduledBack, err := m.isEngineScheduledOnNode(instanceName)
+			if err != nil {
+				m.logger.WithError(err).WithField("instance", instanceName).Errorf("Failed to check CR scheduled node for engine instance orphan %v", orphanName)
+				instanceCRScheduledBack = false
+			} else {
+				instanceCRScheduledBack = isCRScheduledBack
+			}
+		case longhorn.OrphanTypeReplicaInstance:
+			instanceProc, instanceExist = replicaProcess[instanceName]
+			isCRScheduledBack, err := m.isReplicaScheduledOnNode(instanceName)
+			if err != nil {
+				m.logger.WithError(err).WithField("instance", instanceName).Errorf("Failed to check CR scheduled node for replica instance orphan %v", orphanName)
+				instanceCRScheduledBack = false
+			} else {
+				instanceCRScheduledBack = isCRScheduledBack
+			}
+		}
+		var instanceState longhorn.InstanceState
+		if instanceExist {
+			instanceState = instanceProc.Status.State
+		} else {
+			instanceState = longhorn.InstanceStateTerminated
+		}
+
+		if orphan.DeletionTimestamp.IsZero() {
+			if !instanceExist || instanceCRScheduledBack {
+				m.logger.WithFields(logrus.Fields{
+					"instanceExist":           instanceExist,
+					"instanceState":           instanceState,
+					"instanceCRScheduledBack": instanceCRScheduledBack,
+				}).Debugf("Orphan instance %s is deletable", orphanName)
+				deleteOrphans[orphanName] = true
+			}
+		}
+	}
+
+	// delete orphan CRs
+	autoDeletionTypes, err := m.ds.GetSettingOrphanResourceAutoDeletion()
+	var autoDeleteEngine = false
+	var autoDeleteReplica = false
+	if err != nil {
+		m.logger.WithError(err).Warnf("Failed to fetch orphan auto deletion setting, disable by default")
+	} else {
+		autoDeleteEngine = autoDeletionTypes[types.OrphanResourceTypeEngineInstance]
+		autoDeleteReplica = autoDeletionTypes[types.OrphanResourceTypeReplicaInstance]
+	}
+	m.cleanupOrphans(existOrphans, deleteOrphans, autoDeleteEngine, autoDeleteReplica)
+}
+
+func (m *InstanceManagerMonitor) getNodeRuntimeOrphanMap() (map[string]*longhorn.Orphan, error) {
+	existOrphans, err := m.ds.ListOrphansByNode(m.controllerID)
+	if err != nil {
+		return nil, err
+	}
+	orphanMap := make(map[string]*longhorn.Orphan)
+	for _, orphan := range existOrphans {
+		switch orphan.Spec.Type {
+		case longhorn.OrphanTypeEngineInstance, longhorn.OrphanTypeReplicaInstance:
+			orphanMap[orphan.Name] = orphan
+		}
+	}
+	return orphanMap, nil
+}
+
+func (m *InstanceManagerMonitor) isEngineOrphaned(instanceName string) (bool, error) {
+	existEngine, err := m.ds.GetEngineRO(instanceName)
+	switch {
+	case err == nil:
+		// Engine CR still exists - check the ownership
+		return m.isInstanceOrphanedOnNode(&existEngine.Spec.InstanceSpec, &existEngine.Status.InstanceStatus), nil
+	case apierrors.IsNotFound(err):
+		// Engine CR not found - instance is orphaned
+		return true, nil
+	default:
+		// Unexpected error - unable to check if engine instance is orphaned or not
+		return false, err
+	}
+}
+
+func (m *InstanceManagerMonitor) isReplicaOrphaned(instanceName string) (bool, error) {
+	existReplica, err := m.ds.GetReplicaRO(instanceName)
+	switch {
+	case err == nil:
+		// Replica CR still exists - check the ownership
+		return m.isInstanceOrphanedOnNode(&existReplica.Spec.InstanceSpec, &existReplica.Status.InstanceStatus), nil
+	case apierrors.IsNotFound(err):
+		// Replica CR not found - instance is orphaned
+		return true, nil
+	default:
+		// Unexpected error - unable to check if replica instance is orphaned or not
+		return false, err
+	}
+}
+
+// isInstanceOrphanedOnNode returns true only when it is very certain that an instance is scheduled on another node
+func (m *InstanceManagerMonitor) isInstanceOrphanedOnNode(spec *longhorn.InstanceSpec, status *longhorn.InstanceStatus) bool {
+	if status.CurrentState != spec.DesireState {
+		return false
+	}
+	switch status.CurrentState {
+	case longhorn.InstanceStateRunning:
+		return status.OwnerID == spec.NodeID && spec.NodeID != m.controllerID
+	case longhorn.InstanceStateStopped:
+		return spec.NodeID != m.controllerID
+	default:
+		return false
+	}
+}
+
+func (m *InstanceManagerMonitor) isEngineScheduledOnNode(instanceName string) (bool, error) {
+	existEngine, err := m.ds.GetEngineRO(instanceName)
+	switch {
+	case err == nil:
+		// Engine CR still exists - check the ownership
+		return m.isInstanceScheduledOnNode(&existEngine.Spec.InstanceSpec, &existEngine.Status.InstanceStatus), nil
+	case apierrors.IsNotFound(err):
+		// Engine CR not found - instance is orphaned
+		return false, nil
+	default:
+		// Unexpected error - unable to check if engine instance is orphaned or not
+		return false, err
+	}
+}
+
+func (m *InstanceManagerMonitor) isReplicaScheduledOnNode(instanceName string) (bool, error) {
+	existReplica, err := m.ds.GetReplicaRO(instanceName)
+	switch {
+	case err == nil:
+		// Replica CR still exists - check the ownership
+		return m.isInstanceScheduledOnNode(&existReplica.Spec.InstanceSpec, &existReplica.Status.InstanceStatus), nil
+	case apierrors.IsNotFound(err):
+		// Replica CR not found - instance is orphaned
+		return false, nil
+	default:
+		// Unexpected error - unable to check if replica instance is orphaned or not
+		return false, err
+	}
+}
+
+// isInstanceScheduledOnNode returns true only when it is very certain that an instance is scheduled on this node
+func (m *InstanceManagerMonitor) isInstanceScheduledOnNode(spec *longhorn.InstanceSpec, status *longhorn.InstanceStatus) bool {
+	if status.CurrentState != spec.DesireState {
+		return false
+	}
+	switch status.CurrentState {
+	case longhorn.InstanceStateRunning:
+		return status.OwnerID == spec.NodeID && spec.NodeID == m.controllerID
+	case longhorn.InstanceStateStopped:
+		return spec.NodeID == m.controllerID
+	default:
+		return false
+	}
+}
+
+func (m *InstanceManagerMonitor) createOrphan(im *longhorn.InstanceManager, instanceName string, orphanType longhorn.OrphanType, dataEngineType longhorn.DataEngineType) error {
+	name := types.GetOrphanChecksumNameForOrphanedInstance(instanceName, m.controllerID, string(dataEngineType))
+
+	if _, err := m.ds.GetOrphanRO(name); err == nil || !apierrors.IsNotFound(err) {
+		return err
+	}
+
+	// labels will be attached by mutator webhook
+	orphan := &longhorn.Orphan{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:            name,
+			OwnerReferences: datastore.GetOwnerReferencesForInstanceManager(im),
+		},
+		Spec: longhorn.OrphanSpec{
+			NodeID:     m.controllerID,
+			Type:       orphanType,
+			DataEngine: dataEngineType,
+			Parameters: map[string]string{
+				longhorn.OrphanInstanceName:    instanceName,
+				longhorn.OrphanInstanceManager: im.Name,
+			},
+		},
+	}
+
+	_, err := m.ds.CreateOrphan(orphan)
+	return err
+}
+
+func (m *InstanceManagerMonitor) cleanupOrphans(existOrphans map[string]*longhorn.Orphan, deleteOrphans map[string]bool, autoDeleteEngine, autoDeleteReplica bool) {
+	for orphanName, orphan := range existOrphans {
+		instanceName := orphan.Spec.Parameters[longhorn.OrphanInstanceName]
+		switch orphan.Spec.Type {
+		case longhorn.OrphanTypeEngineInstance:
+			if err := m.cleanupOrphan(orphanName, deleteOrphans, autoDeleteEngine); err != nil {
+				m.logger.WithError(err).WithField("instance", instanceName).Errorf("Failed to cleanup engine orphan %s", orphanName)
+			}
+		case longhorn.OrphanTypeReplicaInstance:
+			if err := m.cleanupOrphan(orphanName, deleteOrphans, autoDeleteReplica); err != nil {
+				m.logger.WithError(err).WithField("instance", instanceName).Errorf("Failed to cleanup replica orphan %s", orphanName)
+			}
+		}
+	}
+}
+
+func (m *InstanceManagerMonitor) cleanupOrphan(orphanName string, deleteOrphans map[string]bool, autoDelete bool) error {
path: controller/instance_manager_controller.go
user: login: derekbit
body: nit: why not `deleteOrphan`?
```suggestion
func (m *InstanceManagerMonitor) deleteOrphan(orphanName string, deleteOrphans map[string]bool, autoDelete bool) error {
```
created_at: 2025-04-22T07:54:07Z
author_association: MEMBER
diff_hunk: @@ -2044,6 +2087,265 @@ func (m *InstanceManagerMonitor) StopMonitorWithLock() {
 	m.done = true
 }
 
+func (m *InstanceManagerMonitor) syncOrphans(im *longhorn.InstanceManager, instanceProcess map[string]longhorn.InstanceProcess) {
+	engineProcess := map[string]longhorn.InstanceProcess{}
+	replicaProcess := map[string]longhorn.InstanceProcess{}
+	for name, process := range instanceProcess {
+		switch process.Status.Type {
+		case longhorn.InstanceTypeEngine:
+			engineProcess[name] = process
+		case longhorn.InstanceTypeReplica:
+			replicaProcess[name] = process
+		}
+	}
+
+	existOrphans, err := m.getNodeRuntimeOrphanMap()
+	if err != nil {
+		m.logger.WithError(err).Errorf("Failed to list orphans on node %s", m.controllerID)
+		return
+	}
+
+	// create orphan CR for orphaned instances
+	for _, engineInstance := range engineProcess {
+		if isOrphaned, err := m.isEngineOrphaned(engineInstance.Spec.Name); err != nil {
+			m.logger.WithError(err).Errorf("Failed to check orphan for engine instance %v", engineInstance.Spec.Name)
+		} else if isOrphaned {
+			m.logger.Infof("Engine instance %v is orphaned", engineInstance.Spec.Name)
+			err := m.createOrphan(im, engineInstance.Spec.Name, longhorn.OrphanTypeEngineInstance, engineInstance.Spec.DataEngine)
+			if err != nil {
+				m.logger.WithError(err).Errorf("Failed to create orphan for engine instance %v", engineInstance.Spec.Name)
+			}
+		}
+	}
+	for _, replicaInstance := range replicaProcess {
+		if isOrphaned, err := m.isReplicaOrphaned(replicaInstance.Spec.Name); err != nil {
+			m.logger.WithError(err).Errorf("Failed to check orphan for replica instance %v", replicaInstance.Spec.Name)
+		} else if isOrphaned {
+			m.logger.Infof("Replica instance %v is orphaned", replicaInstance.Spec.Name)
+			err := m.createOrphan(im, replicaInstance.Spec.Name, longhorn.OrphanTypeReplicaInstance, replicaInstance.Spec.DataEngine)
+			if err != nil {
+				m.logger.WithError(err).Errorf("Failed to create orphan for replica instance %v", replicaInstance.Spec.Name)
+			}
+		}
+	}
+
+	// update instance state on orphan CRs
+	deleteOrphans := make(map[string]bool)
+	for orphanName, orphan := range existOrphans {
+		instanceName := orphan.Spec.Parameters[longhorn.OrphanInstanceName]
+		var instanceProc longhorn.InstanceProcess
+		var instanceExist, instanceCRScheduledBack bool
+		switch orphan.Spec.Type {
+		case longhorn.OrphanTypeEngineInstance:
+			instanceProc, instanceExist = engineProcess[instanceName]
+			isCRScheduledBack, err := m.isEngineScheduledOnNode(instanceName)
+			if err != nil {
+				m.logger.WithError(err).WithField("instance", instanceName).Errorf("Failed to check CR scheduled node for engine instance orphan %v", orphanName)
+				instanceCRScheduledBack = false
+			} else {
+				instanceCRScheduledBack = isCRScheduledBack
+			}
+		case longhorn.OrphanTypeReplicaInstance:
+			instanceProc, instanceExist = replicaProcess[instanceName]
+			isCRScheduledBack, err := m.isReplicaScheduledOnNode(instanceName)
+			if err != nil {
+				m.logger.WithError(err).WithField("instance", instanceName).Errorf("Failed to check CR scheduled node for replica instance orphan %v", orphanName)
+				instanceCRScheduledBack = false
+			} else {
+				instanceCRScheduledBack = isCRScheduledBack
+			}
+		}
+		var instanceState longhorn.InstanceState
+		if instanceExist {
+			instanceState = instanceProc.Status.State
+		} else {
+			instanceState = longhorn.InstanceStateTerminated
+		}
+
+		if orphan.DeletionTimestamp.IsZero() {
+			if !instanceExist || instanceCRScheduledBack {
+				m.logger.WithFields(logrus.Fields{
+					"instanceExist":           instanceExist,
+					"instanceState":           instanceState,
+					"instanceCRScheduledBack": instanceCRScheduledBack,
+				}).Debugf("Orphan instance %s is deletable", orphanName)
+				deleteOrphans[orphanName] = true
+			}
+		}
+	}
+
+	// delete orphan CRs
+	autoDeletionTypes, err := m.ds.GetSettingOrphanResourceAutoDeletion()
+	var autoDeleteEngine = false
+	var autoDeleteReplica = false
+	if err != nil {
+		m.logger.WithError(err).Warnf("Failed to fetch orphan auto deletion setting, disable by default")
+	} else {
+		autoDeleteEngine = autoDeletionTypes[types.OrphanResourceTypeEngineInstance]
+		autoDeleteReplica = autoDeletionTypes[types.OrphanResourceTypeReplicaInstance]
+	}
+	m.cleanupOrphans(existOrphans, deleteOrphans, autoDeleteEngine, autoDeleteReplica)
+}
+
+func (m *InstanceManagerMonitor) getNodeRuntimeOrphanMap() (map[string]*longhorn.Orphan, error) {
+	existOrphans, err := m.ds.ListOrphansByNode(m.controllerID)
+	if err != nil {
+		return nil, err
+	}
+	orphanMap := make(map[string]*longhorn.Orphan)
+	for _, orphan := range existOrphans {
+		switch orphan.Spec.Type {
+		case longhorn.OrphanTypeEngineInstance, longhorn.OrphanTypeReplicaInstance:
+			orphanMap[orphan.Name] = orphan
+		}
+	}
+	return orphanMap, nil
+}
+
+func (m *InstanceManagerMonitor) isEngineOrphaned(instanceName string) (bool, error) {
+	existEngine, err := m.ds.GetEngineRO(instanceName)
+	switch {
+	case err == nil:
+		// Engine CR still exists - check the ownership
+		return m.isInstanceOrphanedOnNode(&existEngine.Spec.InstanceSpec, &existEngine.Status.InstanceStatus), nil
+	case apierrors.IsNotFound(err):
+		// Engine CR not found - instance is orphaned
+		return true, nil
+	default:
+		// Unexpected error - unable to check if engine instance is orphaned or not
+		return false, err
+	}
+}
+
+func (m *InstanceManagerMonitor) isReplicaOrphaned(instanceName string) (bool, error) {
+	existReplica, err := m.ds.GetReplicaRO(instanceName)
+	switch {
+	case err == nil:
+		// Replica CR still exists - check the ownership
+		return m.isInstanceOrphanedOnNode(&existReplica.Spec.InstanceSpec, &existReplica.Status.InstanceStatus), nil
+	case apierrors.IsNotFound(err):
+		// Replica CR not found - instance is orphaned
+		return true, nil
+	default:
+		// Unexpected error - unable to check if replica instance is orphaned or not
+		return false, err
+	}
+}
+
+// isInstanceOrphanedOnNode returns true only when it is very certain that an instance is scheduled on another node
+func (m *InstanceManagerMonitor) isInstanceOrphanedOnNode(spec *longhorn.InstanceSpec, status *longhorn.InstanceStatus) bool {
+	if status.CurrentState != spec.DesireState {
+		return false
+	}
+	switch status.CurrentState {
+	case longhorn.InstanceStateRunning:
+		return status.OwnerID == spec.NodeID && spec.NodeID != m.controllerID
+	case longhorn.InstanceStateStopped:
+		return spec.NodeID != m.controllerID
+	default:
+		return false
+	}
+}
+
+func (m *InstanceManagerMonitor) isEngineScheduledOnNode(instanceName string) (bool, error) {
+	existEngine, err := m.ds.GetEngineRO(instanceName)
+	switch {
+	case err == nil:
+		// Engine CR still exists - check the ownership
+		return m.isInstanceScheduledOnNode(&existEngine.Spec.InstanceSpec, &existEngine.Status.InstanceStatus), nil
+	case apierrors.IsNotFound(err):
+		// Engine CR not found - instance is orphaned
+		return false, nil
+	default:
+		// Unexpected error - unable to check if engine instance is orphaned or not
+		return false, err
+	}
+}
+
+func (m *InstanceManagerMonitor) isReplicaScheduledOnNode(instanceName string) (bool, error) {
+	existReplica, err := m.ds.GetReplicaRO(instanceName)
+	switch {
+	case err == nil:
+		// Replica CR still exists - check the ownership
+		return m.isInstanceScheduledOnNode(&existReplica.Spec.InstanceSpec, &existReplica.Status.InstanceStatus), nil
+	case apierrors.IsNotFound(err):
+		// Replica CR not found - instance is orphaned
+		return false, nil
+	default:
+		// Unexpected error - unable to check if replica instance is orphaned or not
+		return false, err
+	}
+}
+
+// isInstanceScheduledOnNode returns true only when it is very certain that an instance is scheduled on this node
+func (m *InstanceManagerMonitor) isInstanceScheduledOnNode(spec *longhorn.InstanceSpec, status *longhorn.InstanceStatus) bool {
+	if status.CurrentState != spec.DesireState {
+		return false
+	}
+	switch status.CurrentState {
+	case longhorn.InstanceStateRunning:
+		return status.OwnerID == spec.NodeID && spec.NodeID == m.controllerID
+	case longhorn.InstanceStateStopped:
+		return spec.NodeID == m.controllerID
+	default:
+		return false
+	}
+}
+
+func (m *InstanceManagerMonitor) createOrphan(im *longhorn.InstanceManager, instanceName string, orphanType longhorn.OrphanType, dataEngineType longhorn.DataEngineType) error {
+	name := types.GetOrphanChecksumNameForOrphanedInstance(instanceName, m.controllerID, string(dataEngineType))
+
+	if _, err := m.ds.GetOrphanRO(name); err == nil || !apierrors.IsNotFound(err) {
+		return err
+	}
+
+	// labels will be attached by mutator webhook
+	orphan := &longhorn.Orphan{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:            name,
+			OwnerReferences: datastore.GetOwnerReferencesForInstanceManager(im),
+		},
+		Spec: longhorn.OrphanSpec{
+			NodeID:     m.controllerID,
+			Type:       orphanType,
+			DataEngine: dataEngineType,
+			Parameters: map[string]string{
+				longhorn.OrphanInstanceName:    instanceName,
+				longhorn.OrphanInstanceManager: im.Name,
+			},
+		},
+	}
+
+	_, err := m.ds.CreateOrphan(orphan)
+	return err
+}
+
+func (m *InstanceManagerMonitor) cleanupOrphans(existOrphans map[string]*longhorn.Orphan, deleteOrphans map[string]bool, autoDeleteEngine, autoDeleteReplica bool) {
path: controller/instance_manager_controller.go
user: login: derekbit
body: nit: why not `deleteOrphans`?
```suggestion
func (m *InstanceManagerMonitor) deleteOrphans(existOrphans map[string]*longhorn.Orphan, deleteOrphans map[string]bool, autoDeleteEngine, autoDeleteReplica bool) {
```
created_at: 2025-04-22T07:54:16Z
author_association: MEMBER
diff_hunk: @@ -2044,6 +2087,265 @@ func (m *InstanceManagerMonitor) StopMonitorWithLock() {
 	m.done = true
 }
 
+func (m *InstanceManagerMonitor) syncOrphans(im *longhorn.InstanceManager, instanceProcess map[string]longhorn.InstanceProcess) {
+	engineProcess := map[string]longhorn.InstanceProcess{}
+	replicaProcess := map[string]longhorn.InstanceProcess{}
+	for name, process := range instanceProcess {
+		switch process.Status.Type {
+		case longhorn.InstanceTypeEngine:
+			engineProcess[name] = process
+		case longhorn.InstanceTypeReplica:
+			replicaProcess[name] = process
+		}
+	}
+
+	existOrphans, err := m.getNodeRuntimeOrphanMap()
+	if err != nil {
+		m.logger.WithError(err).Errorf("Failed to list orphans on node %s", m.controllerID)
+		return
+	}
+
+	// create orphan CR for orphaned instances
+	for _, engineInstance := range engineProcess {
+		if isOrphaned, err := m.isEngineOrphaned(engineInstance.Spec.Name); err != nil {
+			m.logger.WithError(err).Errorf("Failed to check orphan for engine instance %v", engineInstance.Spec.Name)
+		} else if isOrphaned {
+			m.logger.Infof("Engine instance %v is orphaned", engineInstance.Spec.Name)
+			err := m.createOrphan(im, engineInstance.Spec.Name, longhorn.OrphanTypeEngineInstance, engineInstance.Spec.DataEngine)
+			if err != nil {
+				m.logger.WithError(err).Errorf("Failed to create orphan for engine instance %v", engineInstance.Spec.Name)
+			}
+		}
+	}
+	for _, replicaInstance := range replicaProcess {
+		if isOrphaned, err := m.isReplicaOrphaned(replicaInstance.Spec.Name); err != nil {
+			m.logger.WithError(err).Errorf("Failed to check orphan for replica instance %v", replicaInstance.Spec.Name)
+		} else if isOrphaned {
+			m.logger.Infof("Replica instance %v is orphaned", replicaInstance.Spec.Name)
+			err := m.createOrphan(im, replicaInstance.Spec.Name, longhorn.OrphanTypeReplicaInstance, replicaInstance.Spec.DataEngine)
+			if err != nil {
+				m.logger.WithError(err).Errorf("Failed to create orphan for replica instance %v", replicaInstance.Spec.Name)
+			}
+		}
+	}
+
+	// update instance state on orphan CRs
+	deleteOrphans := make(map[string]bool)
+	for orphanName, orphan := range existOrphans {
+		instanceName := orphan.Spec.Parameters[longhorn.OrphanInstanceName]
+		var instanceProc longhorn.InstanceProcess
+		var instanceExist, instanceCRScheduledBack bool
+		switch orphan.Spec.Type {
+		case longhorn.OrphanTypeEngineInstance:
+			instanceProc, instanceExist = engineProcess[instanceName]
+			isCRScheduledBack, err := m.isEngineScheduledOnNode(instanceName)
+			if err != nil {
+				m.logger.WithError(err).WithField("instance", instanceName).Errorf("Failed to check CR scheduled node for engine instance orphan %v", orphanName)
+				instanceCRScheduledBack = false
+			} else {
+				instanceCRScheduledBack = isCRScheduledBack
+			}
+		case longhorn.OrphanTypeReplicaInstance:
+			instanceProc, instanceExist = replicaProcess[instanceName]
+			isCRScheduledBack, err := m.isReplicaScheduledOnNode(instanceName)
+			if err != nil {
+				m.logger.WithError(err).WithField("instance", instanceName).Errorf("Failed to check CR scheduled node for replica instance orphan %v", orphanName)
+				instanceCRScheduledBack = false
+			} else {
+				instanceCRScheduledBack = isCRScheduledBack
+			}
+		}
+		var instanceState longhorn.InstanceState
+		if instanceExist {
+			instanceState = instanceProc.Status.State
+		} else {
+			instanceState = longhorn.InstanceStateTerminated
+		}
+
+		if orphan.DeletionTimestamp.IsZero() {
+			if !instanceExist || instanceCRScheduledBack {
+				m.logger.WithFields(logrus.Fields{
+					"instanceExist":           instanceExist,
+					"instanceState":           instanceState,
+					"instanceCRScheduledBack": instanceCRScheduledBack,
+				}).Debugf("Orphan instance %s is deletable", orphanName)
+				deleteOrphans[orphanName] = true
+			}
+		}
+	}
+
+	// delete orphan CRs
+	autoDeletionTypes, err := m.ds.GetSettingOrphanResourceAutoDeletion()
+	var autoDeleteEngine = false
+	var autoDeleteReplica = false
+	if err != nil {
+		m.logger.WithError(err).Warnf("Failed to fetch orphan auto deletion setting, disable by default")
+	} else {
+		autoDeleteEngine = autoDeletionTypes[types.OrphanResourceTypeEngineInstance]
+		autoDeleteReplica = autoDeletionTypes[types.OrphanResourceTypeReplicaInstance]
+	}
+	m.cleanupOrphans(existOrphans, deleteOrphans, autoDeleteEngine, autoDeleteReplica)
+}
+
+func (m *InstanceManagerMonitor) getNodeRuntimeOrphanMap() (map[string]*longhorn.Orphan, error) {
+	existOrphans, err := m.ds.ListOrphansByNode(m.controllerID)
+	if err != nil {
+		return nil, err
+	}
+	orphanMap := make(map[string]*longhorn.Orphan)
+	for _, orphan := range existOrphans {
+		switch orphan.Spec.Type {
+		case longhorn.OrphanTypeEngineInstance, longhorn.OrphanTypeReplicaInstance:
+			orphanMap[orphan.Name] = orphan
+		}
+	}
+	return orphanMap, nil
+}
+
+func (m *InstanceManagerMonitor) isEngineOrphaned(instanceName string) (bool, error) {
+	existEngine, err := m.ds.GetEngineRO(instanceName)
+	switch {
+	case err == nil:
+		// Engine CR still exists - check the ownership
+		return m.isInstanceOrphanedOnNode(&existEngine.Spec.InstanceSpec, &existEngine.Status.InstanceStatus), nil
+	case apierrors.IsNotFound(err):
+		// Engine CR not found - instance is orphaned
+		return true, nil
+	default:
+		// Unexpected error - unable to check if engine instance is orphaned or not
+		return false, err
+	}
+}
+
+func (m *InstanceManagerMonitor) isReplicaOrphaned(instanceName string) (bool, error) {
+	existReplica, err := m.ds.GetReplicaRO(instanceName)
+	switch {
+	case err == nil:
+		// Replica CR still exists - check the ownership
+		return m.isInstanceOrphanedOnNode(&existReplica.Spec.InstanceSpec, &existReplica.Status.InstanceStatus), nil
+	case apierrors.IsNotFound(err):
+		// Replica CR not found - instance is orphaned
+		return true, nil
+	default:
+		// Unexpected error - unable to check if replica instance is orphaned or not
+		return false, err
+	}
+}
+
+// isInstanceOrphanedOnNode returns true only when it is very certain that an instance is scheduled on another node
+func (m *InstanceManagerMonitor) isInstanceOrphanedOnNode(spec *longhorn.InstanceSpec, status *longhorn.InstanceStatus) bool {
+	if status.CurrentState != spec.DesireState {
+		return false
+	}
+	switch status.CurrentState {
+	case longhorn.InstanceStateRunning:
+		return status.OwnerID == spec.NodeID && spec.NodeID != m.controllerID
+	case longhorn.InstanceStateStopped:
+		return spec.NodeID != m.controllerID
+	default:
+		return false
+	}
+}
+
+func (m *InstanceManagerMonitor) isEngineScheduledOnNode(instanceName string) (bool, error) {
+	existEngine, err := m.ds.GetEngineRO(instanceName)
+	switch {
+	case err == nil:
+		// Engine CR still exists - check the ownership
+		return m.isInstanceScheduledOnNode(&existEngine.Spec.InstanceSpec, &existEngine.Status.InstanceStatus), nil
+	case apierrors.IsNotFound(err):
+		// Engine CR not found - instance is orphaned
+		return false, nil
+	default:
+		// Unexpected error - unable to check if engine instance is orphaned or not
+		return false, err
+	}
+}
+
+func (m *InstanceManagerMonitor) isReplicaScheduledOnNode(instanceName string) (bool, error) {
+	existReplica, err := m.ds.GetReplicaRO(instanceName)
+	switch {
+	case err == nil:
+		// Replica CR still exists - check the ownership
+		return m.isInstanceScheduledOnNode(&existReplica.Spec.InstanceSpec, &existReplica.Status.InstanceStatus), nil
+	case apierrors.IsNotFound(err):
+		// Replica CR not found - instance is orphaned
+		return false, nil
+	default:
+		// Unexpected error - unable to check if replica instance is orphaned or not
+		return false, err
+	}
+}
+
+// isInstanceScheduledOnNode returns true only when it is very certain that an instance is scheduled on this node
+func (m *InstanceManagerMonitor) isInstanceScheduledOnNode(spec *longhorn.InstanceSpec, status *longhorn.InstanceStatus) bool {
+	if status.CurrentState != spec.DesireState {
+		return false
+	}
+	switch status.CurrentState {
+	case longhorn.InstanceStateRunning:
+		return status.OwnerID == spec.NodeID && spec.NodeID == m.controllerID
+	case longhorn.InstanceStateStopped:
+		return spec.NodeID == m.controllerID
+	default:
+		return false
+	}
+}
+
+func (m *InstanceManagerMonitor) createOrphan(im *longhorn.InstanceManager, instanceName string, orphanType longhorn.OrphanType, dataEngineType longhorn.DataEngineType) error {
+	name := types.GetOrphanChecksumNameForOrphanedInstance(instanceName, m.controllerID, string(dataEngineType))
+
+	if _, err := m.ds.GetOrphanRO(name); err == nil || !apierrors.IsNotFound(err) {
+		return err
+	}
+
+	// labels will be attached by mutator webhook
+	orphan := &longhorn.Orphan{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:            name,
+			OwnerReferences: datastore.GetOwnerReferencesForInstanceManager(im),
+		},
+		Spec: longhorn.OrphanSpec{
+			NodeID:     m.controllerID,
+			Type:       orphanType,
+			DataEngine: dataEngineType,
+			Parameters: map[string]string{
+				longhorn.OrphanInstanceName:    instanceName,
+				longhorn.OrphanInstanceManager: im.Name,
+			},
+		},
+	}
+
+	_, err := m.ds.CreateOrphan(orphan)
+	return err
+}
+
+func (m *InstanceManagerMonitor) cleanupOrphans(existOrphans map[string]*longhorn.Orphan, deleteOrphans map[string]bool, autoDeleteEngine, autoDeleteReplica bool) {
+	for orphanName, orphan := range existOrphans {
+		instanceName := orphan.Spec.Parameters[longhorn.OrphanInstanceName]
+		switch orphan.Spec.Type {
+		case longhorn.OrphanTypeEngineInstance:
+			if err := m.cleanupOrphan(orphanName, deleteOrphans, autoDeleteEngine); err != nil {
+				m.logger.WithError(err).WithField("instance", instanceName).Errorf("Failed to cleanup engine orphan %s", orphanName)
+			}
+		case longhorn.OrphanTypeReplicaInstance:
+			if err := m.cleanupOrphan(orphanName, deleteOrphans, autoDeleteReplica); err != nil {
+				m.logger.WithError(err).WithField("instance", instanceName).Errorf("Failed to cleanup replica orphan %s", orphanName)
+			}
+		}
+	}
+}
+
+func (m *InstanceManagerMonitor) cleanupOrphan(orphanName string, deleteOrphans map[string]bool, autoDelete bool) error {
+	if autoDelete {
+		return m.ds.DeleteOrphan(orphanName)
+	}
+	if toDelete := deleteOrphans[orphanName]; toDelete {
+		return m.ds.DeleteOrphan(orphanName)
+	}
path: controller/instance_manager_controller.go
user: login: derekbit
body: How about simplifying the codes

```
if autoDelete || deleteOrphans[orphanName] {
    return m.ds.DeleteOrphan(orphanName)
}
```
created_at: 2025-04-22T07:56:38Z
author_association: MEMBER
diff_hunk: @@ -85,3 +101,49 @@ func upgradeVolumes(namespace string, lhClient *lhclientset.Clientset, resourceM
 
 	return nil
 }
+
+func updateSettings(namespace string, lhClient *lhclientset.Clientset, kubeClient *clientset.Clientset, resourceMaps map[string]interface{}) (err error) {
+	defer func() {
+		err = errors.Wrapf(err, upgradeLogPrefix+"upgrade settings failed")
+	}()
+
+	settingsMap, err := upgradeutil.ListAndUpdateSettingsInProvidedCache(namespace, lhClient, resourceMaps)
+	if err != nil {
+		if apierrors.IsNotFound(err) {
+			return nil
+		}
+		return errors.Wrapf(err, "failed to list all existing Longhorn settings during the settings upgrade")
+	}
path: upgrade/v18xto190/upgrade.go
user: login: COLDTURNIP
body: Will remove it.
created_at: 2025-04-22T10:28:56Z
author_association: MEMBER
diff_hunk: @@ -227,13 +263,11 @@ func (oc *OrphanController) reconcile(orphanName string) (err error) {
 	}
 
 	if !orphan.DeletionTimestamp.IsZero() {
-		defer func() {
-			if err == nil {
-				err = oc.ds.RemoveFinalizerForOrphan(orphan)
-			}
-		}()
-
-		return oc.cleanupOrphanedData(orphan)
+		isCleanupComplete, err := oc.cleanupOrphanedResource(orphan)
path: controller/orphan_controller.go
user: login: COLDTURNIP
body: Instance manager may take time to wait for instance terminating. Orphan controller will confirm the instance state with instance manager after sending the deletion request. `isCleanupComplete` will be false if the instance is still exist (terminating or some other state), and the deletion will continue after the instance state changes, triggered by instance manager status update event.
created_at: 2025-04-22T10:37:13Z
author_association: MEMBER
diff_hunk: @@ -371,6 +572,55 @@ func (oc *OrphanController) updateConditions(orphan *longhorn.Orphan) error {
 	return nil
 }
 
+// If the InstanceState condition status is true, the corresponding instance is still exist in instance manager
+func (oc *OrphanController) updateInstanceStateCondition(orphan *longhorn.Orphan, instanceType longhorn.InstanceType) (err error) {
+	var instanceState longhorn.InstanceState
+	defer func() {
+		if err != nil {
+			return
+		}
+
+		status := longhorn.ConditionStatusTrue
+		if instanceState == longhorn.InstanceStateTerminated {
+			status = longhorn.ConditionStatusFalse
+		}
+
+		orphan.Status.Conditions = types.SetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeInstanceState, status, string(instanceState), "")
+	}()
+
+	instanceName, instanceManager, err := oc.extractOrphanedInstanceInfo(orphan)
+	im, err := oc.ds.GetInstanceManager(instanceManager)
+	if err != nil {
+		if !apierrors.IsNotFound(err) {
+			oc.logger.WithError(err).Infof("No instance manager for node %v for update instance state of orphan instance %v", oc.controllerID, orphan.Name)
+			instanceState = longhorn.InstanceStateTerminated
+			return nil
+		}
+		return err
+	}
+	imc, err := engineapi.NewInstanceManagerClient(im, false)
+	if err != nil {
+		return errors.Wrapf(err, "failed to check node down or missing manager for node %v", orphan.Spec.NodeID)
+	}
+	defer func() {
+		if closeErr := imc.Close(); closeErr != nil {
+			oc.logger.WithError(closeErr).Error("failed to close instance manager client")
+		}
+	}()
+
+	instanceProc, err := imc.InstanceGet(orphan.Spec.DataEngine, instanceName, string(instanceType))
+	switch {
+	case err != nil:
+		return errors.Wrapf(err, "failed to get instance %v", instanceName)
+	case instanceProc == nil:
+		instanceState = longhorn.InstanceStateTerminated
+	default:
+		instanceState = instanceProc.Status.State
+	}
path: controller/orphan_controller.go
user: login: COLDTURNIP
body: To track the state of corresponding instance, and make trouble shooting easier. User don't need to compare the orphan object with the instance list in im CR.
created_at: 2025-04-22T10:46:00Z
author_association: MEMBER
diff_hunk: @@ -2044,6 +2087,265 @@ func (m *InstanceManagerMonitor) StopMonitorWithLock() {
 	m.done = true
 }
 
+func (m *InstanceManagerMonitor) syncOrphans(im *longhorn.InstanceManager, instanceProcess map[string]longhorn.InstanceProcess) {
+	engineProcess := map[string]longhorn.InstanceProcess{}
+	replicaProcess := map[string]longhorn.InstanceProcess{}
+	for name, process := range instanceProcess {
+		switch process.Status.Type {
+		case longhorn.InstanceTypeEngine:
+			engineProcess[name] = process
+		case longhorn.InstanceTypeReplica:
+			replicaProcess[name] = process
+		}
+	}
+
+	existOrphans, err := m.getNodeRuntimeOrphanMap()
+	if err != nil {
+		m.logger.WithError(err).Errorf("Failed to list orphans on node %s", m.controllerID)
+		return
+	}
+
+	// create orphan CR for orphaned instances
+	for _, engineInstance := range engineProcess {
+		if isOrphaned, err := m.isEngineOrphaned(engineInstance.Spec.Name); err != nil {
+			m.logger.WithError(err).Errorf("Failed to check orphan for engine instance %v", engineInstance.Spec.Name)
+		} else if isOrphaned {
+			m.logger.Infof("Engine instance %v is orphaned", engineInstance.Spec.Name)
+			err := m.createOrphan(im, engineInstance.Spec.Name, longhorn.OrphanTypeEngineInstance, engineInstance.Spec.DataEngine)
+			if err != nil {
+				m.logger.WithError(err).Errorf("Failed to create orphan for engine instance %v", engineInstance.Spec.Name)
+			}
+		}
+	}
+	for _, replicaInstance := range replicaProcess {
+		if isOrphaned, err := m.isReplicaOrphaned(replicaInstance.Spec.Name); err != nil {
+			m.logger.WithError(err).Errorf("Failed to check orphan for replica instance %v", replicaInstance.Spec.Name)
+		} else if isOrphaned {
+			m.logger.Infof("Replica instance %v is orphaned", replicaInstance.Spec.Name)
+			err := m.createOrphan(im, replicaInstance.Spec.Name, longhorn.OrphanTypeReplicaInstance, replicaInstance.Spec.DataEngine)
+			if err != nil {
+				m.logger.WithError(err).Errorf("Failed to create orphan for replica instance %v", replicaInstance.Spec.Name)
+			}
+		}
+	}
+
+	// update instance state on orphan CRs
+	deleteOrphans := make(map[string]bool)
+	for orphanName, orphan := range existOrphans {
+		instanceName := orphan.Spec.Parameters[longhorn.OrphanInstanceName]
+		var instanceProc longhorn.InstanceProcess
+		var instanceExist, instanceCRScheduledBack bool
+		switch orphan.Spec.Type {
+		case longhorn.OrphanTypeEngineInstance:
+			instanceProc, instanceExist = engineProcess[instanceName]
+			isCRScheduledBack, err := m.isEngineScheduledOnNode(instanceName)
+			if err != nil {
+				m.logger.WithError(err).WithField("instance", instanceName).Errorf("Failed to check CR scheduled node for engine instance orphan %v", orphanName)
+				instanceCRScheduledBack = false
+			} else {
+				instanceCRScheduledBack = isCRScheduledBack
+			}
+		case longhorn.OrphanTypeReplicaInstance:
+			instanceProc, instanceExist = replicaProcess[instanceName]
+			isCRScheduledBack, err := m.isReplicaScheduledOnNode(instanceName)
+			if err != nil {
+				m.logger.WithError(err).WithField("instance", instanceName).Errorf("Failed to check CR scheduled node for replica instance orphan %v", orphanName)
+				instanceCRScheduledBack = false
+			} else {
+				instanceCRScheduledBack = isCRScheduledBack
+			}
+		}
+		var instanceState longhorn.InstanceState
+		if instanceExist {
+			instanceState = instanceProc.Status.State
+		} else {
+			instanceState = longhorn.InstanceStateTerminated
+		}
+
+		if orphan.DeletionTimestamp.IsZero() {
+			if !instanceExist || instanceCRScheduledBack {
+				m.logger.WithFields(logrus.Fields{
+					"instanceExist":           instanceExist,
+					"instanceState":           instanceState,
+					"instanceCRScheduledBack": instanceCRScheduledBack,
+				}).Debugf("Orphan instance %s is deletable", orphanName)
+				deleteOrphans[orphanName] = true
+			}
+		}
+	}
+
+	// delete orphan CRs
+	autoDeletionTypes, err := m.ds.GetSettingOrphanResourceAutoDeletion()
+	var autoDeleteEngine = false
+	var autoDeleteReplica = false
+	if err != nil {
+		m.logger.WithError(err).Warnf("Failed to fetch orphan auto deletion setting, disable by default")
path: controller/instance_manager_controller.go
user: login: COLDTURNIP
body: We have similar logic in node controller. Though the settings is broken and auto cleanup is disabled, im monitor still need to remove the orphan CRs that the instance disappeared from im (orphan CRs in `deleteOrphans` in this function).
created_at: 2025-04-22T10:53:05Z
author_association: MEMBER
diff_hunk: @@ -85,3 +101,49 @@ func upgradeVolumes(namespace string, lhClient *lhclientset.Clientset, resourceM
 
 	return nil
 }
+
+func updateSettings(namespace string, lhClient *lhclientset.Clientset, kubeClient *clientset.Clientset, resourceMaps map[string]interface{}) (err error) {
+	defer func() {
+		err = errors.Wrapf(err, upgradeLogPrefix+"upgrade settings failed")
+	}()
+
+	settingsMap, err := upgradeutil.ListAndUpdateSettingsInProvidedCache(namespace, lhClient, resourceMaps)
+	if err != nil {
+		if apierrors.IsNotFound(err) {
+			return nil
+		}
+		return errors.Wrapf(err, "failed to list all existing Longhorn settings during the settings upgrade")
+	}
path: upgrade/v18xto190/upgrade.go
user: login: COLDTURNIP
body: We have `upgradeutil.ListAndUpdateVolumesInProvidedCache` in `upgradeVolumes`, and actually `upgradeutil.ListAndUpdate*InProvidedCache` returns the existing loaded CRs if presents in cache. Should we avoid invoking these loader multiple times?
created_at: 2025-04-22T11:02:49Z
author_association: MEMBER
diff_hunk: @@ -85,3 +101,49 @@ func upgradeVolumes(namespace string, lhClient *lhclientset.Clientset, resourceM
 
 	return nil
 }
+
+func updateSettings(namespace string, lhClient *lhclientset.Clientset, kubeClient *clientset.Clientset, resourceMaps map[string]interface{}) (err error) {
+	defer func() {
+		err = errors.Wrapf(err, upgradeLogPrefix+"upgrade settings failed")
+	}()
+
+	settingsMap, err := upgradeutil.ListAndUpdateSettingsInProvidedCache(namespace, lhClient, resourceMaps)
+	if err != nil {
+		if apierrors.IsNotFound(err) {
+			return nil
+		}
+		return errors.Wrapf(err, "failed to list all existing Longhorn settings during the settings upgrade")
+	}
path: upgrade/v18xto190/upgrade.go
user: login: derekbit
body: Okay. Got your point. Then, this should be fine.
created_at: 2025-04-22T11:35:18Z
author_association: MEMBER
diff_hunk: @@ -32,6 +36,18 @@ func UpgradeResources(namespace string, lhClient *lhclientset.Clientset, kubeCli
 		return errors.New("resourceMaps cannot be nil")
 	}
 
+	if err := updateCRs(namespace, lhClient, kubeClient, resourceMaps); err != nil {
+		return err
+	}
+
+	if err := updateSettings(namespace, lhClient, kubeClient, resourceMaps); err != nil {
path: upgrade/v18xto190/upgrade.go
user: login: COLDTURNIP
body: Changed to `upgradeSettings` as a part of `updateCRs`
created_at: 2025-04-22T16:17:11Z
author_association: MEMBER
diff_hunk: @@ -1610,6 +1630,22 @@ func (imc *InstanceManagerController) createInstanceManagerPodSpec(im *longhorn.
 	return podSpec, nil
 }
 
+func (imc *InstanceManagerController) cleanupInstanceManagerOrphanInstances(im *longhorn.InstanceManager) error {
+	orphans, err := imc.ds.ListOrphansByNodeRO(im.Spec.NodeID)
+	if err != nil {
+		return err
+	}
+	for _, orphan := range orphans {
+		switch orphan.Spec.Type {
+		case longhorn.OrphanTypeEngineInstance, longhorn.OrphanTypeReplicaInstance:
+			if err := imc.ds.DeleteOrphan(orphan.Name); err != nil && !apierrors.IsNotFound(err) {
+				return err
+			}
path: controller/instance_manager_controller.go
user: login: COLDTURNIP
body: Refined. Please help review this part again, thanks.
created_at: 2025-04-22T16:19:12Z
author_association: MEMBER
diff_hunk: @@ -2044,6 +2087,265 @@ func (m *InstanceManagerMonitor) StopMonitorWithLock() {
 	m.done = true
 }
 
+func (m *InstanceManagerMonitor) syncOrphans(im *longhorn.InstanceManager, instanceProcess map[string]longhorn.InstanceProcess) {
+	engineProcess := map[string]longhorn.InstanceProcess{}
+	replicaProcess := map[string]longhorn.InstanceProcess{}
+	for name, process := range instanceProcess {
+		switch process.Status.Type {
+		case longhorn.InstanceTypeEngine:
+			engineProcess[name] = process
+		case longhorn.InstanceTypeReplica:
+			replicaProcess[name] = process
+		}
+	}
+
+	existOrphans, err := m.getNodeRuntimeOrphanMap()
+	if err != nil {
+		m.logger.WithError(err).Errorf("Failed to list orphans on node %s", m.controllerID)
+		return
+	}
+
+	// create orphan CR for orphaned instances
+	for _, engineInstance := range engineProcess {
+		if isOrphaned, err := m.isEngineOrphaned(engineInstance.Spec.Name); err != nil {
+			m.logger.WithError(err).Errorf("Failed to check orphan for engine instance %v", engineInstance.Spec.Name)
+		} else if isOrphaned {
+			m.logger.Infof("Engine instance %v is orphaned", engineInstance.Spec.Name)
+			err := m.createOrphan(im, engineInstance.Spec.Name, longhorn.OrphanTypeEngineInstance, engineInstance.Spec.DataEngine)
+			if err != nil {
+				m.logger.WithError(err).Errorf("Failed to create orphan for engine instance %v", engineInstance.Spec.Name)
+			}
+		}
+	}
+	for _, replicaInstance := range replicaProcess {
+		if isOrphaned, err := m.isReplicaOrphaned(replicaInstance.Spec.Name); err != nil {
+			m.logger.WithError(err).Errorf("Failed to check orphan for replica instance %v", replicaInstance.Spec.Name)
+		} else if isOrphaned {
+			m.logger.Infof("Replica instance %v is orphaned", replicaInstance.Spec.Name)
+			err := m.createOrphan(im, replicaInstance.Spec.Name, longhorn.OrphanTypeReplicaInstance, replicaInstance.Spec.DataEngine)
+			if err != nil {
+				m.logger.WithError(err).Errorf("Failed to create orphan for replica instance %v", replicaInstance.Spec.Name)
+			}
+		}
+	}
path: controller/instance_manager_controller.go
user: login: COLDTURNIP
body: Extracted to `createOrphanForInstancesAndUpdateCache`
created_at: 2025-04-22T16:22:51Z
author_association: MEMBER
diff_hunk: @@ -2044,6 +2087,265 @@ func (m *InstanceManagerMonitor) StopMonitorWithLock() {
 	m.done = true
 }
 
+func (m *InstanceManagerMonitor) syncOrphans(im *longhorn.InstanceManager, instanceProcess map[string]longhorn.InstanceProcess) {
+	engineProcess := map[string]longhorn.InstanceProcess{}
+	replicaProcess := map[string]longhorn.InstanceProcess{}
+	for name, process := range instanceProcess {
+		switch process.Status.Type {
+		case longhorn.InstanceTypeEngine:
+			engineProcess[name] = process
+		case longhorn.InstanceTypeReplica:
+			replicaProcess[name] = process
+		}
+	}
+
+	existOrphans, err := m.getNodeRuntimeOrphanMap()
+	if err != nil {
+		m.logger.WithError(err).Errorf("Failed to list orphans on node %s", m.controllerID)
+		return
+	}
+
+	// create orphan CR for orphaned instances
+	for _, engineInstance := range engineProcess {
+		if isOrphaned, err := m.isEngineOrphaned(engineInstance.Spec.Name); err != nil {
+			m.logger.WithError(err).Errorf("Failed to check orphan for engine instance %v", engineInstance.Spec.Name)
+		} else if isOrphaned {
+			m.logger.Infof("Engine instance %v is orphaned", engineInstance.Spec.Name)
+			err := m.createOrphan(im, engineInstance.Spec.Name, longhorn.OrphanTypeEngineInstance, engineInstance.Spec.DataEngine)
+			if err != nil {
+				m.logger.WithError(err).Errorf("Failed to create orphan for engine instance %v", engineInstance.Spec.Name)
+			}
+		}
+	}
+	for _, replicaInstance := range replicaProcess {
+		if isOrphaned, err := m.isReplicaOrphaned(replicaInstance.Spec.Name); err != nil {
+			m.logger.WithError(err).Errorf("Failed to check orphan for replica instance %v", replicaInstance.Spec.Name)
+		} else if isOrphaned {
+			m.logger.Infof("Replica instance %v is orphaned", replicaInstance.Spec.Name)
+			err := m.createOrphan(im, replicaInstance.Spec.Name, longhorn.OrphanTypeReplicaInstance, replicaInstance.Spec.DataEngine)
+			if err != nil {
+				m.logger.WithError(err).Errorf("Failed to create orphan for replica instance %v", replicaInstance.Spec.Name)
+			}
+		}
+	}
+
+	// update instance state on orphan CRs
+	deleteOrphans := make(map[string]bool)
+	for orphanName, orphan := range existOrphans {
+		instanceName := orphan.Spec.Parameters[longhorn.OrphanInstanceName]
+		var instanceProc longhorn.InstanceProcess
+		var instanceExist, instanceCRScheduledBack bool
+		switch orphan.Spec.Type {
+		case longhorn.OrphanTypeEngineInstance:
+			instanceProc, instanceExist = engineProcess[instanceName]
+			isCRScheduledBack, err := m.isEngineScheduledOnNode(instanceName)
+			if err != nil {
+				m.logger.WithError(err).WithField("instance", instanceName).Errorf("Failed to check CR scheduled node for engine instance orphan %v", orphanName)
+				instanceCRScheduledBack = false
+			} else {
+				instanceCRScheduledBack = isCRScheduledBack
+			}
+		case longhorn.OrphanTypeReplicaInstance:
+			instanceProc, instanceExist = replicaProcess[instanceName]
+			isCRScheduledBack, err := m.isReplicaScheduledOnNode(instanceName)
+			if err != nil {
+				m.logger.WithError(err).WithField("instance", instanceName).Errorf("Failed to check CR scheduled node for replica instance orphan %v", orphanName)
+				instanceCRScheduledBack = false
+			} else {
+				instanceCRScheduledBack = isCRScheduledBack
+			}
+		}
+		var instanceState longhorn.InstanceState
+		if instanceExist {
+			instanceState = instanceProc.Status.State
+		} else {
+			instanceState = longhorn.InstanceStateTerminated
+		}
+
+		if orphan.DeletionTimestamp.IsZero() {
+			if !instanceExist || instanceCRScheduledBack {
+				m.logger.WithFields(logrus.Fields{
+					"instanceExist":           instanceExist,
+					"instanceState":           instanceState,
+					"instanceCRScheduledBack": instanceCRScheduledBack,
+				}).Debugf("Orphan instance %s is deletable", orphanName)
+				deleteOrphans[orphanName] = true
+			}
+		}
+	}
path: controller/instance_manager_controller.go
user: login: COLDTURNIP
body: Extracted to `getDeletableOrphans`
created_at: 2025-04-22T16:24:10Z
author_association: MEMBER
diff_hunk: @@ -1771,6 +1799,32 @@ func UnmarshalNodeSelector(nodeSelectorSetting string) (map[string]string, error
 	return nodeSelector, nil
 }
 
+func UnmarshalOrphanResourceTypes(resourceTypesSetting string) (map[OrphanResourceType]bool, error) {
path: types/setting.go
user: login: COLDTURNIP
body: Will cover this.
created_at: 2025-04-22T16:25:56Z
author_association: MEMBER
diff_hunk: @@ -85,3 +101,49 @@ func upgradeVolumes(namespace string, lhClient *lhclientset.Clientset, resourceM
 
 	return nil
 }
+
+func updateSettings(namespace string, lhClient *lhclientset.Clientset, kubeClient *clientset.Clientset, resourceMaps map[string]interface{}) (err error) {
+	defer func() {
+		err = errors.Wrapf(err, upgradeLogPrefix+"upgrade settings failed")
+	}()
+
+	settingsMap, err := upgradeutil.ListAndUpdateSettingsInProvidedCache(namespace, lhClient, resourceMaps)
+	if err != nil {
+		if apierrors.IsNotFound(err) {
+			return nil
+		}
+		return errors.Wrapf(err, "failed to list all existing Longhorn settings during the settings upgrade")
+	}
+
+	if err := updateOrphanResourceAutoDeletionSetting(settingsMap); err != nil {
+		return errors.Wrapf(err, "failed to update orphan resource auto-deletion settings")
+	}
+
+	return nil
+}
+
+func updateOrphanResourceAutoDeletionSetting(settingsMap map[string]*v1beta2.Setting) (err error) {
+	oldSetting, exist := settingsMap[string(types.SettingNameOrphanAutoDeletion)]
+	if !exist {
+		return nil
+	}
+	newSetting, exist := settingsMap[string(types.SettingNameOrphanResourceAutoDeletion)]
+	if !exist {
+		newSetting = &v1beta2.Setting{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: string(types.SettingNameOrphanResourceAutoDeletion),
+			},
+		}
+		settingsMap[string(types.SettingNameOrphanResourceAutoDeletion)] = newSetting
+	}
+	resourceTypes, _ := types.UnmarshalOrphanResourceTypes(newSetting.Value)
+	resourceTypes[types.OrphanResourceTypeReplicaData] = oldSetting.Value == "true"
+	enabledResourceType := make([]string, 0, len(resourceTypes))
+	for rt, enabled := range resourceTypes {
+		if enabled {
+			enabledResourceType = append(enabledResourceType, string(rt))
+		}
+	}
+	newSetting.Value = strings.Join(enabledResourceType, ";")
+	return nil
+}
path: upgrade/v18xto190/upgrade.go
user: login: COLDTURNIP
body: Will improve the test coverage.
created_at: 2025-04-22T16:26:17Z
author_association: MEMBER
diff_hunk: @@ -2044,6 +2087,265 @@ func (m *InstanceManagerMonitor) StopMonitorWithLock() {
 	m.done = true
 }
 
+func (m *InstanceManagerMonitor) syncOrphans(im *longhorn.InstanceManager, instanceProcess map[string]longhorn.InstanceProcess) {
+	engineProcess := map[string]longhorn.InstanceProcess{}
+	replicaProcess := map[string]longhorn.InstanceProcess{}
+	for name, process := range instanceProcess {
+		switch process.Status.Type {
+		case longhorn.InstanceTypeEngine:
+			engineProcess[name] = process
+		case longhorn.InstanceTypeReplica:
+			replicaProcess[name] = process
+		}
+	}
+
+	existOrphans, err := m.getNodeRuntimeOrphanMap()
+	if err != nil {
+		m.logger.WithError(err).Errorf("Failed to list orphans on node %s", m.controllerID)
+		return
+	}
+
+	// create orphan CR for orphaned instances
+	for _, engineInstance := range engineProcess {
+		if isOrphaned, err := m.isEngineOrphaned(engineInstance.Spec.Name); err != nil {
+			m.logger.WithError(err).Errorf("Failed to check orphan for engine instance %v", engineInstance.Spec.Name)
+		} else if isOrphaned {
+			m.logger.Infof("Engine instance %v is orphaned", engineInstance.Spec.Name)
+			err := m.createOrphan(im, engineInstance.Spec.Name, longhorn.OrphanTypeEngineInstance, engineInstance.Spec.DataEngine)
+			if err != nil {
+				m.logger.WithError(err).Errorf("Failed to create orphan for engine instance %v", engineInstance.Spec.Name)
+			}
+		}
+	}
+	for _, replicaInstance := range replicaProcess {
+		if isOrphaned, err := m.isReplicaOrphaned(replicaInstance.Spec.Name); err != nil {
+			m.logger.WithError(err).Errorf("Failed to check orphan for replica instance %v", replicaInstance.Spec.Name)
+		} else if isOrphaned {
+			m.logger.Infof("Replica instance %v is orphaned", replicaInstance.Spec.Name)
+			err := m.createOrphan(im, replicaInstance.Spec.Name, longhorn.OrphanTypeReplicaInstance, replicaInstance.Spec.DataEngine)
+			if err != nil {
+				m.logger.WithError(err).Errorf("Failed to create orphan for replica instance %v", replicaInstance.Spec.Name)
+			}
+		}
+	}
+
+	// update instance state on orphan CRs
+	deleteOrphans := make(map[string]bool)
+	for orphanName, orphan := range existOrphans {
+		instanceName := orphan.Spec.Parameters[longhorn.OrphanInstanceName]
+		var instanceProc longhorn.InstanceProcess
+		var instanceExist, instanceCRScheduledBack bool
+		switch orphan.Spec.Type {
+		case longhorn.OrphanTypeEngineInstance:
+			instanceProc, instanceExist = engineProcess[instanceName]
+			isCRScheduledBack, err := m.isEngineScheduledOnNode(instanceName)
+			if err != nil {
+				m.logger.WithError(err).WithField("instance", instanceName).Errorf("Failed to check CR scheduled node for engine instance orphan %v", orphanName)
+				instanceCRScheduledBack = false
+			} else {
+				instanceCRScheduledBack = isCRScheduledBack
+			}
+		case longhorn.OrphanTypeReplicaInstance:
+			instanceProc, instanceExist = replicaProcess[instanceName]
+			isCRScheduledBack, err := m.isReplicaScheduledOnNode(instanceName)
+			if err != nil {
+				m.logger.WithError(err).WithField("instance", instanceName).Errorf("Failed to check CR scheduled node for replica instance orphan %v", orphanName)
+				instanceCRScheduledBack = false
+			} else {
+				instanceCRScheduledBack = isCRScheduledBack
+			}
+		}
+		var instanceState longhorn.InstanceState
+		if instanceExist {
+			instanceState = instanceProc.Status.State
+		} else {
+			instanceState = longhorn.InstanceStateTerminated
+		}
+
+		if orphan.DeletionTimestamp.IsZero() {
+			if !instanceExist || instanceCRScheduledBack {
+				m.logger.WithFields(logrus.Fields{
+					"instanceExist":           instanceExist,
+					"instanceState":           instanceState,
+					"instanceCRScheduledBack": instanceCRScheduledBack,
+				}).Debugf("Orphan instance %s is deletable", orphanName)
+				deleteOrphans[orphanName] = true
+			}
+		}
+	}
+
+	// delete orphan CRs
+	autoDeletionTypes, err := m.ds.GetSettingOrphanResourceAutoDeletion()
+	var autoDeleteEngine = false
+	var autoDeleteReplica = false
+	if err != nil {
+		m.logger.WithError(err).Warnf("Failed to fetch orphan auto deletion setting, disable by default")
path: controller/instance_manager_controller.go
user: login: derekbit
body: I see. Thanks for the explanation.
created_at: 2025-04-23T00:40:57Z
author_association: MEMBER
diff_hunk: @@ -36,6 +42,10 @@ type OrphanSpec struct {
 	// Can be "replica".
 	// +optional
 	Type OrphanType `json:"orphanType"`
+	// The type of data engine for instance orphan.
+	// Can be "v1", "v2".
+	// +optional
path: k8s/pkg/apis/longhorn/v1beta2/orphan.go
user: login: derekbit
body: 
```suggestion
	// +kubebuilder:validation:Enum=v1;v2
```
created_at: 2025-04-23T00:49:54Z
author_association: MEMBER
diff_hunk: @@ -2044,6 +2084,268 @@ func (m *InstanceManagerMonitor) StopMonitorWithLock() {
 	m.done = true
 }
 
+func (m *InstanceManagerMonitor) syncOrphans(im *longhorn.InstanceManager, allInstances instanceProcessMap) {
+	engineProcess, replicaProcess := m.categorizeProcesses(allInstances)
path: controller/instance_manager_controller.go
user: login: derekbit
body: 
```suggestion
	engineProcesses, replicaProcesses := m.categorizeProcesses(allInstances)
```
created_at: 2025-04-23T00:53:17Z
author_association: MEMBER
diff_hunk: @@ -2044,6 +2084,268 @@ func (m *InstanceManagerMonitor) StopMonitorWithLock() {
 	m.done = true
 }
 
+func (m *InstanceManagerMonitor) syncOrphans(im *longhorn.InstanceManager, allInstances instanceProcessMap) {
+	engineProcess, replicaProcess := m.categorizeProcesses(allInstances)
+	existOrphans, err := m.listOrphansForEngineAndReplicaInstances(im.Spec.NodeID)
+	if err != nil {
+		m.logger.WithError(err).Errorf("Failed to list orphans on node %s", im.Spec.NodeID)
+		return
+	}
+
+	// exam instances and create orphan CRs
+	existOrphans = m.createOrphanForInstancesAndUpdateCache(existOrphans, im, engineProcess, longhorn.OrphanTypeEngineInstance, m.isEngineOrphaned)
+	existOrphans = m.createOrphanForInstancesAndUpdateCache(existOrphans, im, replicaProcess, longhorn.OrphanTypeReplicaInstance, m.isReplicaOrphaned)
+
+	deletableOrphans := m.getDeletableOrphans(existOrphans, engineProcess, replicaProcess)
+	m.deleteOrphans(existOrphans, deletableOrphans)
+}
+
+func (m *InstanceManagerMonitor) listOrphansForEngineAndReplicaInstances(nodeName string) (map[string]*longhorn.Orphan, error) {
+	existOrphans, err := m.ds.ListOrphansByNode(nodeName)
+	if err != nil {
+		return nil, err
+	}
+	orphanMap := make(map[string]*longhorn.Orphan)
+	for _, orphan := range existOrphans {
+		switch orphan.Spec.Type {
+		case longhorn.OrphanTypeEngineInstance, longhorn.OrphanTypeReplicaInstance:
+			orphanMap[orphan.Name] = orphan
+		}
+	}
+	return orphanMap, nil
+}
+
+func (m *InstanceManagerMonitor) isEngineOrphaned(instanceName string) (bool, error) {
+	existEngine, err := m.ds.GetEngineRO(instanceName)
+	switch {
+	case err == nil:
+		// Engine CR still exists - check the ownership
+		return m.isInstanceOrphanedOnNode(&existEngine.Spec.InstanceSpec, &existEngine.Status.InstanceStatus), nil
+	case apierrors.IsNotFound(err):
+		// Engine CR not found - instance is orphaned
+		return true, nil
+	default:
+		// Unexpected error - unable to check if engine instance is orphaned or not
+		return false, err
+	}
+}
+
+func (m *InstanceManagerMonitor) isReplicaOrphaned(instanceName string) (bool, error) {
+	existReplica, err := m.ds.GetReplicaRO(instanceName)
+	switch {
+	case err == nil:
+		// Replica CR still exists - check the ownership
+		return m.isInstanceOrphanedOnNode(&existReplica.Spec.InstanceSpec, &existReplica.Status.InstanceStatus), nil
+	case apierrors.IsNotFound(err):
+		// Replica CR not found - instance is orphaned
+		return true, nil
+	default:
+		// Unexpected error - unable to check if replica instance is orphaned or not
+		return false, err
+	}
+}
+
+// isInstanceOrphanedOnNode returns true only when it is very certain that an instance is scheduled on another node
+func (m *InstanceManagerMonitor) isInstanceOrphanedOnNode(spec *longhorn.InstanceSpec, status *longhorn.InstanceStatus) bool {
+	if status.CurrentState != spec.DesireState {
+		return false
+	}
+	switch status.CurrentState {
+	case longhorn.InstanceStateRunning:
+		return status.OwnerID == spec.NodeID && spec.NodeID != m.controllerID
+	case longhorn.InstanceStateStopped:
+		return spec.NodeID != m.controllerID
+	default:
+		return false
+	}
+}
+
+func (m *InstanceManagerMonitor) isEngineScheduledOnNode(instanceName string) (bool, error) {
+	existEngine, err := m.ds.GetEngineRO(instanceName)
+	switch {
+	case err == nil:
+		// Engine CR still exists - check the ownership
+		return m.isInstanceScheduledOnNode(&existEngine.Spec.InstanceSpec, &existEngine.Status.InstanceStatus), nil
+	case apierrors.IsNotFound(err):
+		// Engine CR not found - instance is orphaned
+		return false, nil
+	default:
+		// Unexpected error - unable to check if engine instance is orphaned or not
+		return false, err
+	}
+}
+
+func (m *InstanceManagerMonitor) isReplicaScheduledOnNode(instanceName string) (bool, error) {
+	existReplica, err := m.ds.GetReplicaRO(instanceName)
+	switch {
+	case err == nil:
+		// Replica CR still exists - check the ownership
+		return m.isInstanceScheduledOnNode(&existReplica.Spec.InstanceSpec, &existReplica.Status.InstanceStatus), nil
+	case apierrors.IsNotFound(err):
+		// Replica CR not found - instance is orphaned
+		return false, nil
+	default:
+		// Unexpected error - unable to check if replica instance is orphaned or not
+		return false, err
+	}
+}
+
+// isInstanceScheduledOnNode returns true only when it is very certain that an instance is scheduled on this node
+func (m *InstanceManagerMonitor) isInstanceScheduledOnNode(spec *longhorn.InstanceSpec, status *longhorn.InstanceStatus) bool {
+	if status.CurrentState != spec.DesireState {
+		return false
+	}
+	switch status.CurrentState {
+	case longhorn.InstanceStateRunning:
+		return status.OwnerID == spec.NodeID && spec.NodeID == m.controllerID
+	case longhorn.InstanceStateStopped:
+		return spec.NodeID == m.controllerID
+	default:
+		return false
+	}
+}
+
+func (m *InstanceManagerMonitor) createOrphanForInstancesAndUpdateCache(existOrphans map[string]*longhorn.Orphan, im *longhorn.InstanceManager, instanceMap instanceProcessMap, orphanType longhorn.OrphanType, orphanFilter func(string) (bool, error)) map[string]*longhorn.Orphan {
+	for instanceName, instance := range instanceMap {
+		if _, isExist := existOrphans[instanceName]; isExist {
+			continue
+		}
+		if isOrphaned, err := orphanFilter(instanceName); err != nil {
+			m.logger.WithError(err).Errorf("Failed to check %v orphan for instance %v", orphanType, instanceName)
+		} else if isOrphaned {
+			m.logger.Infof("Instance %v is orphaned", instanceName)
+			newOrphan, err := m.createOrphan(im, instanceName, orphanType, instance.Spec.DataEngine)
+			if err != nil {
+				m.logger.WithError(err).Errorf("Failed to create %v orphan for instance %v", orphanType, instanceName)
+			} else if newOrphan != nil {
+				existOrphans[newOrphan.Name] = newOrphan
+			}
+		}
+	}
+	return existOrphans
+}
+
+func (m *InstanceManagerMonitor) createOrphan(im *longhorn.InstanceManager, instanceName string, orphanType longhorn.OrphanType, dataEngineType longhorn.DataEngineType) (*longhorn.Orphan, error) {
+	name := types.GetOrphanChecksumNameForOrphanedInstance(instanceName, m.controllerID, string(dataEngineType))
+
+	if _, err := m.ds.GetOrphanRO(name); err == nil || !apierrors.IsNotFound(err) {
+		return nil, err
+	}
+
+	// labels will be attached by mutator webhook
+	orphan := &longhorn.Orphan{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:            name,
+			OwnerReferences: datastore.GetOwnerReferencesForInstanceManager(im),
+		},
+		Spec: longhorn.OrphanSpec{
+			NodeID:     m.controllerID,
+			Type:       orphanType,
+			DataEngine: dataEngineType,
+			Parameters: map[string]string{
+				longhorn.OrphanInstanceName:    instanceName,
+				longhorn.OrphanInstanceManager: im.Name,
+			},
+		},
+	}
+
+	return m.ds.CreateOrphan(orphan)
+}
+
+func (m *InstanceManagerMonitor) getDeletableOrphans(existOrphans map[string]*longhorn.Orphan, engineProcess instanceProcessMap, replicaProcess instanceProcessMap) map[string]bool {
+	deletableOrphans := make(map[string]bool)
+	for orphanName, orphan := range existOrphans {
+		instanceName := orphan.Spec.Parameters[longhorn.OrphanInstanceName]
+		var instanceProc longhorn.InstanceProcess
+		var instanceExist, instanceCRScheduledBack bool
+		switch orphan.Spec.Type {
+		case longhorn.OrphanTypeEngineInstance:
+			instanceProc, instanceExist = engineProcess[instanceName]
+			isCRScheduledBack, err := m.isEngineScheduledOnNode(instanceName)
+			if err != nil {
+				m.logger.WithError(err).WithField("instance", instanceName).Errorf("Failed to check CR scheduled node for engine instance orphan %v", orphanName)
+				instanceCRScheduledBack = false
+			} else {
+				instanceCRScheduledBack = isCRScheduledBack
+			}
+		case longhorn.OrphanTypeReplicaInstance:
+			instanceProc, instanceExist = replicaProcess[instanceName]
+			isCRScheduledBack, err := m.isReplicaScheduledOnNode(instanceName)
+			if err != nil {
+				m.logger.WithError(err).WithField("instance", instanceName).Errorf("Failed to check CR scheduled node for replica instance orphan %v", orphanName)
+				instanceCRScheduledBack = false
+			} else {
+				instanceCRScheduledBack = isCRScheduledBack
+			}
+		}
+		var instanceState longhorn.InstanceState
+		if instanceExist {
+			instanceState = instanceProc.Status.State
+		} else {
+			instanceState = longhorn.InstanceStateTerminated
+		}
+
+		if orphan.DeletionTimestamp.IsZero() {
+			if !instanceExist || instanceCRScheduledBack {
+				m.logger.WithFields(logrus.Fields{
+					"instanceExist":           instanceExist,
+					"instanceState":           instanceState,
+					"instanceCRScheduledBack": instanceCRScheduledBack,
+				}).Debugf("Orphan instance %s is deletable", orphanName)
+				deletableOrphans[orphanName] = true
+			}
+		}
+	}
+	return deletableOrphans
+}
+
+func (m *InstanceManagerMonitor) deleteOrphans(instanceOrphans map[string]*longhorn.Orphan, deletableOrphans map[string]bool) {
+	autoDeletionTypes, err := m.ds.GetSettingOrphanResourceAutoDeletion()
+	var autoDeleteEngine = false
+	var autoDeleteReplica = false
+	if err != nil {
+		m.logger.WithError(err).Warnf("Failed to fetch orphan auto deletion setting, disable by default")
path: controller/instance_manager_controller.go
user: login: derekbit
body: 
```suggestion
		m.logger.WithError(err).Warnf("Failed to fetch orphan auto deletion setting, disabled by default")
```
created_at: 2025-04-23T00:58:07Z
author_association: MEMBER
diff_hunk: @@ -269,48 +303,200 @@ func (oc *OrphanController) isResponsibleFor(orphan *longhorn.Orphan) bool {
 	return isControllerResponsibleFor(oc.controllerID, oc.ds, orphan.Name, orphan.Spec.NodeID, orphan.Status.OwnerID)
 }
 
-func (oc *OrphanController) cleanupOrphanedData(orphan *longhorn.Orphan) (err error) {
+func (oc *OrphanController) cleanupOrphanedResource(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
 	log := getLoggerForOrphan(oc.logger, orphan)
 
 	defer func() {
 		if err == nil {
 			return
 		}
 
-		err = errors.Wrapf(err, "failed to delete orphan %v data", orphan.Name)
+		err = errors.Wrapf(err, "failed to delete %v orphan %v", orphan.Spec.Type, orphan.Name)
 		orphan.Status.Conditions = types.SetCondition(orphan.Status.Conditions,
 			longhorn.OrphanConditionTypeError, longhorn.ConditionStatusTrue, "", err.Error())
 	}()
 
 	// Make sure if the orphan nodeID and controller ID are same.
 	// If NO, just delete the orphan resource object and don't touch the data.
 	if orphan.Spec.NodeID != oc.controllerID {
-		log.Infof("Orphan nodeID %v is different from controllerID %v, so just delete the orphan resource object",
-			orphan.Name, oc.controllerID)
-		return nil
+		log.WithFields(logrus.Fields{
+			"orphanType": orphan.Spec.Type,
+			"orphanName": orphan.Name,
+			"orphanNode": orphan.Spec.NodeID,
+		}).Infof("Orphan does not belong to this controller. Skipping resource cleanup")
+		return true, nil
 	}
 
-	if types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeDataCleanable).Status !=
-		longhorn.ConditionStatusTrue {
-		log.Infof("Only delete orphan %v resource object and do not delete the orphaned data", orphan.Name)
-		return nil
+	switch orphan.Spec.Type {
+	case longhorn.OrphanTypeEngineInstance:
+		isCleanupComplete, err = oc.cleanupOrphanedEngineInstance(orphan)
+	case longhorn.OrphanTypeReplicaInstance:
+		isCleanupComplete, err = oc.cleanupOrphanedReplicaInstance(orphan)
+	case longhorn.OrphanTypeReplicaData:
+		if types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeDataCleanable).Status !=
+			longhorn.ConditionStatusTrue {
+			log.Infof("Only delete orphan %v resource object and do not delete the orphaned data store", orphan.Name)
+			return true, nil
+		}
+		err = oc.deleteOrphanedReplicaDataStore(orphan)
+		if err == nil || (err != nil && apierrors.IsNotFound(err)) {
+			isCleanupComplete = true
+			err = nil
+		}
+	default:
+		return false, fmt.Errorf("unknown orphan type %v to clean up orphaned resource for %v", orphan.Spec.Type, orphan.Name)
 	}
 
-	switch orphan.Spec.Type {
-	case longhorn.OrphanTypeReplica:
-		err = oc.deleteOrphanedReplica(orphan)
+	return isCleanupComplete, err
+}
+
+func (oc *OrphanController) cleanupOrphanedEngineInstance(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
+	defer func() {
+		if err != nil {
+			err = errors.Wrapf(err, "cannot cleanup orphan engine instance %v", orphan.Name)
+		}
+	}()
+
+	instance, imName, err := oc.extractOrphanedInstanceInfo(orphan)
+	if err != nil {
+		return false, err
+	}
+
+	var spec *longhorn.InstanceSpec
+	if engineCR, err := oc.ds.GetEngineRO(instance); err != nil {
+		if !apierrors.IsNotFound(err) {
+			return false, err
path: controller/orphan_controller.go
user: login: derekbit
body: If the engine is somehow missing, can we continue the deletion?
created_at: 2025-04-23T01:10:07Z
author_association: MEMBER
diff_hunk: @@ -269,48 +303,200 @@ func (oc *OrphanController) isResponsibleFor(orphan *longhorn.Orphan) bool {
 	return isControllerResponsibleFor(oc.controllerID, oc.ds, orphan.Name, orphan.Spec.NodeID, orphan.Status.OwnerID)
 }
 
-func (oc *OrphanController) cleanupOrphanedData(orphan *longhorn.Orphan) (err error) {
+func (oc *OrphanController) cleanupOrphanedResource(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
 	log := getLoggerForOrphan(oc.logger, orphan)
 
 	defer func() {
 		if err == nil {
 			return
 		}
 
-		err = errors.Wrapf(err, "failed to delete orphan %v data", orphan.Name)
+		err = errors.Wrapf(err, "failed to delete %v orphan %v", orphan.Spec.Type, orphan.Name)
 		orphan.Status.Conditions = types.SetCondition(orphan.Status.Conditions,
 			longhorn.OrphanConditionTypeError, longhorn.ConditionStatusTrue, "", err.Error())
 	}()
 
 	// Make sure if the orphan nodeID and controller ID are same.
 	// If NO, just delete the orphan resource object and don't touch the data.
 	if orphan.Spec.NodeID != oc.controllerID {
-		log.Infof("Orphan nodeID %v is different from controllerID %v, so just delete the orphan resource object",
-			orphan.Name, oc.controllerID)
-		return nil
+		log.WithFields(logrus.Fields{
+			"orphanType": orphan.Spec.Type,
+			"orphanName": orphan.Name,
+			"orphanNode": orphan.Spec.NodeID,
+		}).Infof("Orphan does not belong to this controller. Skipping resource cleanup")
+		return true, nil
 	}
 
-	if types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeDataCleanable).Status !=
-		longhorn.ConditionStatusTrue {
-		log.Infof("Only delete orphan %v resource object and do not delete the orphaned data", orphan.Name)
-		return nil
+	switch orphan.Spec.Type {
+	case longhorn.OrphanTypeEngineInstance:
+		isCleanupComplete, err = oc.cleanupOrphanedEngineInstance(orphan)
+	case longhorn.OrphanTypeReplicaInstance:
+		isCleanupComplete, err = oc.cleanupOrphanedReplicaInstance(orphan)
+	case longhorn.OrphanTypeReplicaData:
+		if types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeDataCleanable).Status !=
+			longhorn.ConditionStatusTrue {
+			log.Infof("Only delete orphan %v resource object and do not delete the orphaned data store", orphan.Name)
+			return true, nil
+		}
+		err = oc.deleteOrphanedReplicaDataStore(orphan)
+		if err == nil || (err != nil && apierrors.IsNotFound(err)) {
+			isCleanupComplete = true
+			err = nil
+		}
+	default:
+		return false, fmt.Errorf("unknown orphan type %v to clean up orphaned resource for %v", orphan.Spec.Type, orphan.Name)
 	}
 
-	switch orphan.Spec.Type {
-	case longhorn.OrphanTypeReplica:
-		err = oc.deleteOrphanedReplica(orphan)
+	return isCleanupComplete, err
+}
+
+func (oc *OrphanController) cleanupOrphanedEngineInstance(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
+	defer func() {
+		if err != nil {
+			err = errors.Wrapf(err, "cannot cleanup orphan engine instance %v", orphan.Name)
+		}
+	}()
+
+	instance, imName, err := oc.extractOrphanedInstanceInfo(orphan)
+	if err != nil {
+		return false, err
+	}
+
+	var spec *longhorn.InstanceSpec
+	if engineCR, err := oc.ds.GetEngineRO(instance); err != nil {
+		if !apierrors.IsNotFound(err) {
+			return false, err
+		}
+		spec = nil
+	} else {
+		spec = &engineCR.Spec.InstanceSpec
+	}
+	return oc.cleanupOrphanedInstance(orphan, instance, imName, longhorn.InstanceManagerTypeEngine, orphan.Spec.DataEngine, spec)
+}
+
+func (oc *OrphanController) cleanupOrphanedReplicaInstance(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
+	defer func() {
+		if err != nil {
+			err = errors.Wrapf(err, "cannot cleanup orphan replica instance %v", orphan.Name)
+		}
+	}()
+
+	instance, imName, err := oc.extractOrphanedInstanceInfo(orphan)
+	if err != nil {
+		return false, err
+	}
+
+	var spec *longhorn.InstanceSpec
+	if replicaCR, err := oc.ds.GetReplicaRO(instance); err != nil {
+		if !apierrors.IsNotFound(err) {
+			return false, err
+		}
+		spec = nil
+	} else {
+		spec = &replicaCR.Spec.InstanceSpec
+	}
+	return oc.cleanupOrphanedInstance(orphan, instance, imName, longhorn.InstanceManagerTypeReplica, orphan.Spec.DataEngine, spec)
+}
+
+func (oc *OrphanController) cleanupOrphanedInstance(orphan *longhorn.Orphan, instance, imName string, imType longhorn.InstanceManagerType, deType longhorn.DataEngineType, instanceCRSpec *longhorn.InstanceSpec) (isCleanupComplete bool, err error) {
+	if instanceCRSpec != nil && instanceCRSpec.NodeID == oc.controllerID {
+		// Instance is scheduled back to current node and is no longer considered orphaned.
+		// Finalize the orphan CR directly without terminating the instance process.
path: controller/orphan_controller.go
user: login: derekbit
body: Can we add a log message here?
created_at: 2025-04-23T01:42:29Z
author_association: MEMBER
diff_hunk: @@ -269,48 +303,200 @@ func (oc *OrphanController) isResponsibleFor(orphan *longhorn.Orphan) bool {
 	return isControllerResponsibleFor(oc.controllerID, oc.ds, orphan.Name, orphan.Spec.NodeID, orphan.Status.OwnerID)
 }
 
-func (oc *OrphanController) cleanupOrphanedData(orphan *longhorn.Orphan) (err error) {
+func (oc *OrphanController) cleanupOrphanedResource(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
 	log := getLoggerForOrphan(oc.logger, orphan)
 
 	defer func() {
 		if err == nil {
 			return
 		}
 
-		err = errors.Wrapf(err, "failed to delete orphan %v data", orphan.Name)
+		err = errors.Wrapf(err, "failed to delete %v orphan %v", orphan.Spec.Type, orphan.Name)
 		orphan.Status.Conditions = types.SetCondition(orphan.Status.Conditions,
 			longhorn.OrphanConditionTypeError, longhorn.ConditionStatusTrue, "", err.Error())
 	}()
 
 	// Make sure if the orphan nodeID and controller ID are same.
 	// If NO, just delete the orphan resource object and don't touch the data.
 	if orphan.Spec.NodeID != oc.controllerID {
-		log.Infof("Orphan nodeID %v is different from controllerID %v, so just delete the orphan resource object",
-			orphan.Name, oc.controllerID)
-		return nil
+		log.WithFields(logrus.Fields{
+			"orphanType": orphan.Spec.Type,
+			"orphanName": orphan.Name,
+			"orphanNode": orphan.Spec.NodeID,
+		}).Infof("Orphan does not belong to this controller. Skipping resource cleanup")
+		return true, nil
 	}
 
-	if types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeDataCleanable).Status !=
-		longhorn.ConditionStatusTrue {
-		log.Infof("Only delete orphan %v resource object and do not delete the orphaned data", orphan.Name)
-		return nil
+	switch orphan.Spec.Type {
+	case longhorn.OrphanTypeEngineInstance:
+		isCleanupComplete, err = oc.cleanupOrphanedEngineInstance(orphan)
+	case longhorn.OrphanTypeReplicaInstance:
+		isCleanupComplete, err = oc.cleanupOrphanedReplicaInstance(orphan)
+	case longhorn.OrphanTypeReplicaData:
+		if types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeDataCleanable).Status !=
+			longhorn.ConditionStatusTrue {
+			log.Infof("Only delete orphan %v resource object and do not delete the orphaned data store", orphan.Name)
+			return true, nil
+		}
+		err = oc.deleteOrphanedReplicaDataStore(orphan)
+		if err == nil || (err != nil && apierrors.IsNotFound(err)) {
+			isCleanupComplete = true
+			err = nil
+		}
+	default:
+		return false, fmt.Errorf("unknown orphan type %v to clean up orphaned resource for %v", orphan.Spec.Type, orphan.Name)
 	}
 
-	switch orphan.Spec.Type {
-	case longhorn.OrphanTypeReplica:
-		err = oc.deleteOrphanedReplica(orphan)
+	return isCleanupComplete, err
+}
+
+func (oc *OrphanController) cleanupOrphanedEngineInstance(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
+	defer func() {
+		if err != nil {
+			err = errors.Wrapf(err, "cannot cleanup orphan engine instance %v", orphan.Name)
+		}
+	}()
+
+	instance, imName, err := oc.extractOrphanedInstanceInfo(orphan)
+	if err != nil {
+		return false, err
+	}
+
+	var spec *longhorn.InstanceSpec
+	if engineCR, err := oc.ds.GetEngineRO(instance); err != nil {
+		if !apierrors.IsNotFound(err) {
+			return false, err
+		}
+		spec = nil
+	} else {
+		spec = &engineCR.Spec.InstanceSpec
+	}
+	return oc.cleanupOrphanedInstance(orphan, instance, imName, longhorn.InstanceManagerTypeEngine, orphan.Spec.DataEngine, spec)
+}
+
+func (oc *OrphanController) cleanupOrphanedReplicaInstance(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
+	defer func() {
+		if err != nil {
+			err = errors.Wrapf(err, "cannot cleanup orphan replica instance %v", orphan.Name)
+		}
+	}()
+
+	instance, imName, err := oc.extractOrphanedInstanceInfo(orphan)
+	if err != nil {
+		return false, err
+	}
+
+	var spec *longhorn.InstanceSpec
+	if replicaCR, err := oc.ds.GetReplicaRO(instance); err != nil {
+		if !apierrors.IsNotFound(err) {
+			return false, err
+		}
+		spec = nil
+	} else {
+		spec = &replicaCR.Spec.InstanceSpec
+	}
+	return oc.cleanupOrphanedInstance(orphan, instance, imName, longhorn.InstanceManagerTypeReplica, orphan.Spec.DataEngine, spec)
+}
+
+func (oc *OrphanController) cleanupOrphanedInstance(orphan *longhorn.Orphan, instance, imName string, imType longhorn.InstanceManagerType, deType longhorn.DataEngineType, instanceCRSpec *longhorn.InstanceSpec) (isCleanupComplete bool, err error) {
+	if instanceCRSpec != nil && instanceCRSpec.NodeID == oc.controllerID {
+		// Instance is scheduled back to current node and is no longer considered orphaned.
+		// Finalize the orphan CR directly without terminating the instance process.
+		return true, nil
+	}
+
+	im, err := oc.ds.GetRunningInstanceManagerByNodeRO(oc.controllerID, deType)
+	if err != nil {
+		oc.logger.WithError(err).Infof("No running instance manager for node %v for deleting orphan instance %v", oc.controllerID, orphan.Name)
path: controller/orphan_controller.go
user: login: derekbit
body: 
```suggestion
		oc.logger.WithError(err).Warnf("No running instance manager for node %v for deleting orphan instance %v", oc.controllerID, orphan.Name)
```
created_at: 2025-04-23T01:43:29Z
author_association: MEMBER
diff_hunk: @@ -269,48 +303,200 @@ func (oc *OrphanController) isResponsibleFor(orphan *longhorn.Orphan) bool {
 	return isControllerResponsibleFor(oc.controllerID, oc.ds, orphan.Name, orphan.Spec.NodeID, orphan.Status.OwnerID)
 }
 
-func (oc *OrphanController) cleanupOrphanedData(orphan *longhorn.Orphan) (err error) {
+func (oc *OrphanController) cleanupOrphanedResource(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
 	log := getLoggerForOrphan(oc.logger, orphan)
 
 	defer func() {
 		if err == nil {
 			return
 		}
 
-		err = errors.Wrapf(err, "failed to delete orphan %v data", orphan.Name)
+		err = errors.Wrapf(err, "failed to delete %v orphan %v", orphan.Spec.Type, orphan.Name)
 		orphan.Status.Conditions = types.SetCondition(orphan.Status.Conditions,
 			longhorn.OrphanConditionTypeError, longhorn.ConditionStatusTrue, "", err.Error())
 	}()
 
 	// Make sure if the orphan nodeID and controller ID are same.
 	// If NO, just delete the orphan resource object and don't touch the data.
 	if orphan.Spec.NodeID != oc.controllerID {
-		log.Infof("Orphan nodeID %v is different from controllerID %v, so just delete the orphan resource object",
-			orphan.Name, oc.controllerID)
-		return nil
+		log.WithFields(logrus.Fields{
+			"orphanType": orphan.Spec.Type,
+			"orphanName": orphan.Name,
+			"orphanNode": orphan.Spec.NodeID,
+		}).Infof("Orphan does not belong to this controller. Skipping resource cleanup")
+		return true, nil
 	}
 
-	if types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeDataCleanable).Status !=
-		longhorn.ConditionStatusTrue {
-		log.Infof("Only delete orphan %v resource object and do not delete the orphaned data", orphan.Name)
-		return nil
+	switch orphan.Spec.Type {
+	case longhorn.OrphanTypeEngineInstance:
+		isCleanupComplete, err = oc.cleanupOrphanedEngineInstance(orphan)
+	case longhorn.OrphanTypeReplicaInstance:
+		isCleanupComplete, err = oc.cleanupOrphanedReplicaInstance(orphan)
+	case longhorn.OrphanTypeReplicaData:
+		if types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeDataCleanable).Status !=
+			longhorn.ConditionStatusTrue {
+			log.Infof("Only delete orphan %v resource object and do not delete the orphaned data store", orphan.Name)
+			return true, nil
+		}
+		err = oc.deleteOrphanedReplicaDataStore(orphan)
+		if err == nil || (err != nil && apierrors.IsNotFound(err)) {
+			isCleanupComplete = true
+			err = nil
+		}
+	default:
+		return false, fmt.Errorf("unknown orphan type %v to clean up orphaned resource for %v", orphan.Spec.Type, orphan.Name)
 	}
 
-	switch orphan.Spec.Type {
-	case longhorn.OrphanTypeReplica:
-		err = oc.deleteOrphanedReplica(orphan)
+	return isCleanupComplete, err
+}
+
+func (oc *OrphanController) cleanupOrphanedEngineInstance(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
+	defer func() {
+		if err != nil {
+			err = errors.Wrapf(err, "cannot cleanup orphan engine instance %v", orphan.Name)
+		}
+	}()
+
+	instance, imName, err := oc.extractOrphanedInstanceInfo(orphan)
+	if err != nil {
+		return false, err
+	}
+
+	var spec *longhorn.InstanceSpec
+	if engineCR, err := oc.ds.GetEngineRO(instance); err != nil {
+		if !apierrors.IsNotFound(err) {
+			return false, err
+		}
+		spec = nil
+	} else {
+		spec = &engineCR.Spec.InstanceSpec
+	}
+	return oc.cleanupOrphanedInstance(orphan, instance, imName, longhorn.InstanceManagerTypeEngine, orphan.Spec.DataEngine, spec)
+}
+
+func (oc *OrphanController) cleanupOrphanedReplicaInstance(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
+	defer func() {
+		if err != nil {
+			err = errors.Wrapf(err, "cannot cleanup orphan replica instance %v", orphan.Name)
+		}
+	}()
+
+	instance, imName, err := oc.extractOrphanedInstanceInfo(orphan)
+	if err != nil {
+		return false, err
+	}
+
+	var spec *longhorn.InstanceSpec
+	if replicaCR, err := oc.ds.GetReplicaRO(instance); err != nil {
+		if !apierrors.IsNotFound(err) {
+			return false, err
+		}
+		spec = nil
+	} else {
+		spec = &replicaCR.Spec.InstanceSpec
+	}
+	return oc.cleanupOrphanedInstance(orphan, instance, imName, longhorn.InstanceManagerTypeReplica, orphan.Spec.DataEngine, spec)
+}
+
+func (oc *OrphanController) cleanupOrphanedInstance(orphan *longhorn.Orphan, instance, imName string, imType longhorn.InstanceManagerType, deType longhorn.DataEngineType, instanceCRSpec *longhorn.InstanceSpec) (isCleanupComplete bool, err error) {
+	if instanceCRSpec != nil && instanceCRSpec.NodeID == oc.controllerID {
+		// Instance is scheduled back to current node and is no longer considered orphaned.
+		// Finalize the orphan CR directly without terminating the instance process.
+		return true, nil
+	}
+
+	im, err := oc.ds.GetRunningInstanceManagerByNodeRO(oc.controllerID, deType)
+	if err != nil {
+		oc.logger.WithError(err).Infof("No running instance manager for node %v for deleting orphan instance %v", oc.controllerID, orphan.Name)
+		return true, nil
+	}
+	if im.Name != imName {
+		oc.logger.WithField("instanceManager", imName).Infof("Orphan instance %v is not managed by current running instance manager %v", orphan.Name, im.Name)
path: controller/orphan_controller.go
user: login: derekbit
body: 
```suggestion
		oc.logger.WithField("instanceManager", imName).Warnf("Orphan instance %v is not managed by current running instance manager %v", orphan.Name, im.Name)
```
created_at: 2025-04-23T01:45:02Z
author_association: MEMBER
diff_hunk: @@ -269,48 +303,200 @@ func (oc *OrphanController) isResponsibleFor(orphan *longhorn.Orphan) bool {
 	return isControllerResponsibleFor(oc.controllerID, oc.ds, orphan.Name, orphan.Spec.NodeID, orphan.Status.OwnerID)
 }
 
-func (oc *OrphanController) cleanupOrphanedData(orphan *longhorn.Orphan) (err error) {
+func (oc *OrphanController) cleanupOrphanedResource(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
 	log := getLoggerForOrphan(oc.logger, orphan)
 
 	defer func() {
 		if err == nil {
 			return
 		}
 
-		err = errors.Wrapf(err, "failed to delete orphan %v data", orphan.Name)
+		err = errors.Wrapf(err, "failed to delete %v orphan %v", orphan.Spec.Type, orphan.Name)
 		orphan.Status.Conditions = types.SetCondition(orphan.Status.Conditions,
 			longhorn.OrphanConditionTypeError, longhorn.ConditionStatusTrue, "", err.Error())
 	}()
 
 	// Make sure if the orphan nodeID and controller ID are same.
 	// If NO, just delete the orphan resource object and don't touch the data.
 	if orphan.Spec.NodeID != oc.controllerID {
-		log.Infof("Orphan nodeID %v is different from controllerID %v, so just delete the orphan resource object",
-			orphan.Name, oc.controllerID)
-		return nil
+		log.WithFields(logrus.Fields{
+			"orphanType": orphan.Spec.Type,
+			"orphanName": orphan.Name,
+			"orphanNode": orphan.Spec.NodeID,
+		}).Infof("Orphan does not belong to this controller. Skipping resource cleanup")
+		return true, nil
 	}
 
-	if types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeDataCleanable).Status !=
-		longhorn.ConditionStatusTrue {
-		log.Infof("Only delete orphan %v resource object and do not delete the orphaned data", orphan.Name)
-		return nil
+	switch orphan.Spec.Type {
+	case longhorn.OrphanTypeEngineInstance:
+		isCleanupComplete, err = oc.cleanupOrphanedEngineInstance(orphan)
+	case longhorn.OrphanTypeReplicaInstance:
+		isCleanupComplete, err = oc.cleanupOrphanedReplicaInstance(orphan)
+	case longhorn.OrphanTypeReplicaData:
+		if types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeDataCleanable).Status !=
+			longhorn.ConditionStatusTrue {
+			log.Infof("Only delete orphan %v resource object and do not delete the orphaned data store", orphan.Name)
+			return true, nil
+		}
+		err = oc.deleteOrphanedReplicaDataStore(orphan)
+		if err == nil || (err != nil && apierrors.IsNotFound(err)) {
+			isCleanupComplete = true
+			err = nil
+		}
+	default:
+		return false, fmt.Errorf("unknown orphan type %v to clean up orphaned resource for %v", orphan.Spec.Type, orphan.Name)
 	}
 
-	switch orphan.Spec.Type {
-	case longhorn.OrphanTypeReplica:
-		err = oc.deleteOrphanedReplica(orphan)
+	return isCleanupComplete, err
+}
+
+func (oc *OrphanController) cleanupOrphanedEngineInstance(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
+	defer func() {
+		if err != nil {
+			err = errors.Wrapf(err, "cannot cleanup orphan engine instance %v", orphan.Name)
+		}
+	}()
+
+	instance, imName, err := oc.extractOrphanedInstanceInfo(orphan)
+	if err != nil {
+		return false, err
+	}
+
+	var spec *longhorn.InstanceSpec
+	if engineCR, err := oc.ds.GetEngineRO(instance); err != nil {
+		if !apierrors.IsNotFound(err) {
+			return false, err
+		}
+		spec = nil
+	} else {
+		spec = &engineCR.Spec.InstanceSpec
+	}
+	return oc.cleanupOrphanedInstance(orphan, instance, imName, longhorn.InstanceManagerTypeEngine, orphan.Spec.DataEngine, spec)
+}
+
+func (oc *OrphanController) cleanupOrphanedReplicaInstance(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
+	defer func() {
+		if err != nil {
+			err = errors.Wrapf(err, "cannot cleanup orphan replica instance %v", orphan.Name)
+		}
+	}()
+
+	instance, imName, err := oc.extractOrphanedInstanceInfo(orphan)
+	if err != nil {
+		return false, err
+	}
+
+	var spec *longhorn.InstanceSpec
+	if replicaCR, err := oc.ds.GetReplicaRO(instance); err != nil {
+		if !apierrors.IsNotFound(err) {
+			return false, err
+		}
+		spec = nil
+	} else {
+		spec = &replicaCR.Spec.InstanceSpec
+	}
+	return oc.cleanupOrphanedInstance(orphan, instance, imName, longhorn.InstanceManagerTypeReplica, orphan.Spec.DataEngine, spec)
+}
+
+func (oc *OrphanController) cleanupOrphanedInstance(orphan *longhorn.Orphan, instance, imName string, imType longhorn.InstanceManagerType, deType longhorn.DataEngineType, instanceCRSpec *longhorn.InstanceSpec) (isCleanupComplete bool, err error) {
+	if instanceCRSpec != nil && instanceCRSpec.NodeID == oc.controllerID {
+		// Instance is scheduled back to current node and is no longer considered orphaned.
+		// Finalize the orphan CR directly without terminating the instance process.
+		return true, nil
+	}
+
+	im, err := oc.ds.GetRunningInstanceManagerByNodeRO(oc.controllerID, deType)
+	if err != nil {
+		oc.logger.WithError(err).Infof("No running instance manager for node %v for deleting orphan instance %v", oc.controllerID, orphan.Name)
+		return true, nil
+	}
+	if im.Name != imName {
+		oc.logger.WithField("instanceManager", imName).Infof("Orphan instance %v is not managed by current running instance manager %v", orphan.Name, im.Name)
+		return true, nil
+	}
+	imc, err := engineapi.NewInstanceManagerClient(im, false)
+	if err != nil {
+		return false, err
+	}
+	defer func() {
+		if closeErr := imc.Close(); closeErr != nil {
+			oc.logger.WithError(closeErr).Error("failed to close instance manager client")
path: controller/orphan_controller.go
user: login: derekbit
body: 
```suggestion
			oc.logger.WithError(closeErr).Error("Failed to close instance manager client")
```
created_at: 2025-04-23T01:45:32Z
author_association: MEMBER
diff_hunk: @@ -269,48 +303,200 @@ func (oc *OrphanController) isResponsibleFor(orphan *longhorn.Orphan) bool {
 	return isControllerResponsibleFor(oc.controllerID, oc.ds, orphan.Name, orphan.Spec.NodeID, orphan.Status.OwnerID)
 }
 
-func (oc *OrphanController) cleanupOrphanedData(orphan *longhorn.Orphan) (err error) {
+func (oc *OrphanController) cleanupOrphanedResource(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
 	log := getLoggerForOrphan(oc.logger, orphan)
 
 	defer func() {
 		if err == nil {
 			return
 		}
 
-		err = errors.Wrapf(err, "failed to delete orphan %v data", orphan.Name)
+		err = errors.Wrapf(err, "failed to delete %v orphan %v", orphan.Spec.Type, orphan.Name)
 		orphan.Status.Conditions = types.SetCondition(orphan.Status.Conditions,
 			longhorn.OrphanConditionTypeError, longhorn.ConditionStatusTrue, "", err.Error())
 	}()
 
 	// Make sure if the orphan nodeID and controller ID are same.
 	// If NO, just delete the orphan resource object and don't touch the data.
 	if orphan.Spec.NodeID != oc.controllerID {
-		log.Infof("Orphan nodeID %v is different from controllerID %v, so just delete the orphan resource object",
-			orphan.Name, oc.controllerID)
-		return nil
+		log.WithFields(logrus.Fields{
+			"orphanType": orphan.Spec.Type,
+			"orphanName": orphan.Name,
+			"orphanNode": orphan.Spec.NodeID,
+		}).Infof("Orphan does not belong to this controller. Skipping resource cleanup")
+		return true, nil
 	}
 
-	if types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeDataCleanable).Status !=
-		longhorn.ConditionStatusTrue {
-		log.Infof("Only delete orphan %v resource object and do not delete the orphaned data", orphan.Name)
-		return nil
+	switch orphan.Spec.Type {
+	case longhorn.OrphanTypeEngineInstance:
+		isCleanupComplete, err = oc.cleanupOrphanedEngineInstance(orphan)
+	case longhorn.OrphanTypeReplicaInstance:
+		isCleanupComplete, err = oc.cleanupOrphanedReplicaInstance(orphan)
+	case longhorn.OrphanTypeReplicaData:
+		if types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeDataCleanable).Status !=
+			longhorn.ConditionStatusTrue {
+			log.Infof("Only delete orphan %v resource object and do not delete the orphaned data store", orphan.Name)
+			return true, nil
+		}
+		err = oc.deleteOrphanedReplicaDataStore(orphan)
+		if err == nil || (err != nil && apierrors.IsNotFound(err)) {
+			isCleanupComplete = true
+			err = nil
+		}
+	default:
+		return false, fmt.Errorf("unknown orphan type %v to clean up orphaned resource for %v", orphan.Spec.Type, orphan.Name)
 	}
 
-	switch orphan.Spec.Type {
-	case longhorn.OrphanTypeReplica:
-		err = oc.deleteOrphanedReplica(orphan)
+	return isCleanupComplete, err
+}
+
+func (oc *OrphanController) cleanupOrphanedEngineInstance(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
+	defer func() {
+		if err != nil {
+			err = errors.Wrapf(err, "cannot cleanup orphan engine instance %v", orphan.Name)
+		}
+	}()
+
+	instance, imName, err := oc.extractOrphanedInstanceInfo(orphan)
+	if err != nil {
+		return false, err
+	}
+
+	var spec *longhorn.InstanceSpec
+	if engineCR, err := oc.ds.GetEngineRO(instance); err != nil {
+		if !apierrors.IsNotFound(err) {
+			return false, err
+		}
+		spec = nil
+	} else {
+		spec = &engineCR.Spec.InstanceSpec
+	}
+	return oc.cleanupOrphanedInstance(orphan, instance, imName, longhorn.InstanceManagerTypeEngine, orphan.Spec.DataEngine, spec)
+}
+
+func (oc *OrphanController) cleanupOrphanedReplicaInstance(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
+	defer func() {
+		if err != nil {
+			err = errors.Wrapf(err, "cannot cleanup orphan replica instance %v", orphan.Name)
+		}
+	}()
+
+	instance, imName, err := oc.extractOrphanedInstanceInfo(orphan)
+	if err != nil {
+		return false, err
+	}
+
+	var spec *longhorn.InstanceSpec
+	if replicaCR, err := oc.ds.GetReplicaRO(instance); err != nil {
+		if !apierrors.IsNotFound(err) {
+			return false, err
+		}
+		spec = nil
+	} else {
+		spec = &replicaCR.Spec.InstanceSpec
+	}
+	return oc.cleanupOrphanedInstance(orphan, instance, imName, longhorn.InstanceManagerTypeReplica, orphan.Spec.DataEngine, spec)
+}
+
+func (oc *OrphanController) cleanupOrphanedInstance(orphan *longhorn.Orphan, instance, imName string, imType longhorn.InstanceManagerType, deType longhorn.DataEngineType, instanceCRSpec *longhorn.InstanceSpec) (isCleanupComplete bool, err error) {
+	if instanceCRSpec != nil && instanceCRSpec.NodeID == oc.controllerID {
+		// Instance is scheduled back to current node and is no longer considered orphaned.
+		// Finalize the orphan CR directly without terminating the instance process.
+		return true, nil
+	}
+
+	im, err := oc.ds.GetRunningInstanceManagerByNodeRO(oc.controllerID, deType)
+	if err != nil {
+		oc.logger.WithError(err).Infof("No running instance manager for node %v for deleting orphan instance %v", oc.controllerID, orphan.Name)
+		return true, nil
+	}
+	if im.Name != imName {
+		oc.logger.WithField("instanceManager", imName).Infof("Orphan instance %v is not managed by current running instance manager %v", orphan.Name, im.Name)
+		return true, nil
+	}
+	imc, err := engineapi.NewInstanceManagerClient(im, false)
+	if err != nil {
+		return false, err
+	}
+	defer func() {
+		if closeErr := imc.Close(); closeErr != nil {
+			oc.logger.WithError(closeErr).Error("failed to close instance manager client")
+		}
+	}()
+
+	instanceExist := types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeInstanceExist).Status == longhorn.ConditionStatusTrue
+	if instanceExist {
+		if err = oc.deleteInstance(imc, instance, imType, deType); err != nil {
path: controller/orphan_controller.go
user: login: derekbit
body: 
```suggestion
		if err := oc.deleteInstance(imc, instance, imType, deType); err != nil {
```
created_at: 2025-04-23T01:45:50Z
author_association: MEMBER
diff_hunk: @@ -612,6 +619,12 @@ func (imc *InstanceManagerController) handlePod(im *longhorn.InstanceManager) er
 		return err
 	}
 
+	if isPodDeletedOrNotRunning {
+		if err := imc.deleteOrphans(im); err != nil {
+			log.WithError(err).Warnf("Instance manager %v failed to delete orphan instance CRs for stopped instance manager pod", im.Name)
path: controller/instance_manager_controller.go
user: login: derekbit
body: 
```suggestion
			log.WithError(err).Warnf("Failed to delete orphan CRs for stopped instance manager pod %v", im.Name)
```
created_at: 2025-04-23T02:10:32Z
author_association: MEMBER
diff_hunk: @@ -1995,29 +2046,18 @@ func (m *InstanceManagerMonitor) pollAndUpdateInstanceMap() (needStop bool) {
 	if clusterAutoscalerEnabled && im.Spec.Type == longhorn.InstanceManagerTypeEngine {
 		m.nodeCallback(m.controllerID)
 	}
-
-	return false
 }
 
-func (m *InstanceManagerMonitor) updateInstanceMap(im *longhorn.InstanceManager, resp map[string]longhorn.InstanceProcess) bool {
+func (m *InstanceManagerMonitor) updateInstanceMap(im *longhorn.InstanceManager, instanceMap instanceProcessMap) bool {
 	switch {
 	case im.Status.APIVersion < 4:
-		if reflect.DeepEqual(im.Status.Instances, resp) { // nolint: staticcheck
+		if reflect.DeepEqual(im.Status.Instances, instanceMap) { // nolint: staticcheck
 			return false
 		}
 
-		im.Status.Instances = resp // nolint: staticcheck
+		im.Status.Instances = instanceMap // nolint: staticcheck
 	default:
-		engineProcess := map[string]longhorn.InstanceProcess{}
-		replicaProcess := map[string]longhorn.InstanceProcess{}
-		for name, process := range resp {
-			switch process.Status.Type {
-			case longhorn.InstanceTypeEngine:
-				engineProcess[name] = process
-			case longhorn.InstanceTypeReplica:
-				replicaProcess[name] = process
-			}
-		}
+		engineProcess, replicaProcess := m.categorizeProcesses(instanceMap)
path: controller/instance_manager_controller.go
user: login: derekbit
body: nit
```suggestion
		engineProcesses, replicaProcesses := m.categorizeProcesses(instanceMap)
```
created_at: 2025-04-23T02:17:12Z
author_association: MEMBER
diff_hunk: @@ -1942,48 +1984,57 @@ func (m *InstanceManagerMonitor) Run() {
 			if !needUpdate {
 				continue
 			}
-			if needStop := m.pollAndUpdateInstanceMap(); needStop {
+			instanceManager, instanceMap, needStop := m.pollInstanceMap()
+			if needStop {
 				return
 			}
+			if instanceManager != nil && instanceMap != nil {
+				m.syncInstanceMap(instanceManager, instanceMap)
path: controller/instance_manager_controller.go
user: login: derekbit
body: nit:
```suggestion
				m.syncInstances(instanceManager, instanceMap)
```
created_at: 2025-04-23T02:17:39Z
author_association: MEMBER
diff_hunk: @@ -2044,6 +2084,268 @@ func (m *InstanceManagerMonitor) StopMonitorWithLock() {
 	m.done = true
 }
 
+func (m *InstanceManagerMonitor) syncOrphans(im *longhorn.InstanceManager, allInstances instanceProcessMap) {
+	engineProcess, replicaProcess := m.categorizeProcesses(allInstances)
+	existOrphans, err := m.listOrphansForEngineAndReplicaInstances(im.Spec.NodeID)
+	if err != nil {
+		m.logger.WithError(err).Errorf("Failed to list orphans on node %s", im.Spec.NodeID)
+		return
+	}
+
+	// exam instances and create orphan CRs
+	existOrphans = m.createOrphanForInstancesAndUpdateCache(existOrphans, im, engineProcess, longhorn.OrphanTypeEngineInstance, m.isEngineOrphaned)
+	existOrphans = m.createOrphanForInstancesAndUpdateCache(existOrphans, im, replicaProcess, longhorn.OrphanTypeReplicaInstance, m.isReplicaOrphaned)
+
+	deletableOrphans := m.getDeletableOrphans(existOrphans, engineProcess, replicaProcess)
+	m.deleteOrphans(existOrphans, deletableOrphans)
+}
+
+func (m *InstanceManagerMonitor) listOrphansForEngineAndReplicaInstances(nodeName string) (map[string]*longhorn.Orphan, error) {
+	existOrphans, err := m.ds.ListOrphansByNode(nodeName)
+	if err != nil {
+		return nil, err
+	}
+	orphanMap := make(map[string]*longhorn.Orphan)
+	for _, orphan := range existOrphans {
+		switch orphan.Spec.Type {
+		case longhorn.OrphanTypeEngineInstance, longhorn.OrphanTypeReplicaInstance:
+			orphanMap[orphan.Name] = orphan
+		}
+	}
+	return orphanMap, nil
+}
+
+func (m *InstanceManagerMonitor) isEngineOrphaned(instanceName string) (bool, error) {
+	existEngine, err := m.ds.GetEngineRO(instanceName)
+	switch {
+	case err == nil:
+		// Engine CR still exists - check the ownership
+		return m.isInstanceOrphanedOnNode(&existEngine.Spec.InstanceSpec, &existEngine.Status.InstanceStatus), nil
+	case apierrors.IsNotFound(err):
+		// Engine CR not found - instance is orphaned
+		return true, nil
+	default:
+		// Unexpected error - unable to check if engine instance is orphaned or not
+		return false, err
+	}
+}
+
+func (m *InstanceManagerMonitor) isReplicaOrphaned(instanceName string) (bool, error) {
+	existReplica, err := m.ds.GetReplicaRO(instanceName)
+	switch {
+	case err == nil:
+		// Replica CR still exists - check the ownership
+		return m.isInstanceOrphanedOnNode(&existReplica.Spec.InstanceSpec, &existReplica.Status.InstanceStatus), nil
+	case apierrors.IsNotFound(err):
+		// Replica CR not found - instance is orphaned
+		return true, nil
+	default:
+		// Unexpected error - unable to check if replica instance is orphaned or not
+		return false, err
+	}
+}
+
+// isInstanceOrphanedOnNode returns true only when it is very certain that an instance is scheduled on another node
+func (m *InstanceManagerMonitor) isInstanceOrphanedOnNode(spec *longhorn.InstanceSpec, status *longhorn.InstanceStatus) bool {
+	if status.CurrentState != spec.DesireState {
+		return false
+	}
+	switch status.CurrentState {
+	case longhorn.InstanceStateRunning:
+		return status.OwnerID == spec.NodeID && spec.NodeID != m.controllerID
+	case longhorn.InstanceStateStopped:
+		return spec.NodeID != m.controllerID
+	default:
+		return false
+	}
+}
+
+func (m *InstanceManagerMonitor) isEngineScheduledOnNode(instanceName string) (bool, error) {
+	existEngine, err := m.ds.GetEngineRO(instanceName)
+	switch {
+	case err == nil:
+		// Engine CR still exists - check the ownership
+		return m.isInstanceScheduledOnNode(&existEngine.Spec.InstanceSpec, &existEngine.Status.InstanceStatus), nil
+	case apierrors.IsNotFound(err):
+		// Engine CR not found - instance is orphaned
+		return false, nil
+	default:
+		// Unexpected error - unable to check if engine instance is orphaned or not
+		return false, err
+	}
+}
+
+func (m *InstanceManagerMonitor) isReplicaScheduledOnNode(instanceName string) (bool, error) {
+	existReplica, err := m.ds.GetReplicaRO(instanceName)
+	switch {
+	case err == nil:
+		// Replica CR still exists - check the ownership
+		return m.isInstanceScheduledOnNode(&existReplica.Spec.InstanceSpec, &existReplica.Status.InstanceStatus), nil
+	case apierrors.IsNotFound(err):
+		// Replica CR not found - instance is orphaned
+		return false, nil
+	default:
+		// Unexpected error - unable to check if replica instance is orphaned or not
+		return false, err
+	}
+}
+
+// isInstanceScheduledOnNode returns true only when it is very certain that an instance is scheduled on this node
+func (m *InstanceManagerMonitor) isInstanceScheduledOnNode(spec *longhorn.InstanceSpec, status *longhorn.InstanceStatus) bool {
+	if status.CurrentState != spec.DesireState {
+		return false
+	}
+	switch status.CurrentState {
+	case longhorn.InstanceStateRunning:
+		return status.OwnerID == spec.NodeID && spec.NodeID == m.controllerID
+	case longhorn.InstanceStateStopped:
+		return spec.NodeID == m.controllerID
+	default:
+		return false
+	}
+}
+
+func (m *InstanceManagerMonitor) createOrphanForInstancesAndUpdateCache(existOrphans map[string]*longhorn.Orphan, im *longhorn.InstanceManager, instanceMap instanceProcessMap, orphanType longhorn.OrphanType, orphanFilter func(string) (bool, error)) map[string]*longhorn.Orphan {
+	for instanceName, instance := range instanceMap {
+		if _, isExist := existOrphans[instanceName]; isExist {
+			continue
+		}
+		if isOrphaned, err := orphanFilter(instanceName); err != nil {
+			m.logger.WithError(err).Errorf("Failed to check %v orphan for instance %v", orphanType, instanceName)
+		} else if isOrphaned {
+			m.logger.Infof("Instance %v is orphaned", instanceName)
+			newOrphan, err := m.createOrphan(im, instanceName, orphanType, instance.Spec.DataEngine)
+			if err != nil {
+				m.logger.WithError(err).Errorf("Failed to create %v orphan for instance %v", orphanType, instanceName)
+			} else if newOrphan != nil {
+				existOrphans[newOrphan.Name] = newOrphan
+			}
+		}
+	}
+	return existOrphans
+}
+
+func (m *InstanceManagerMonitor) createOrphan(im *longhorn.InstanceManager, instanceName string, orphanType longhorn.OrphanType, dataEngineType longhorn.DataEngineType) (*longhorn.Orphan, error) {
+	name := types.GetOrphanChecksumNameForOrphanedInstance(instanceName, m.controllerID, string(dataEngineType))
+
+	if _, err := m.ds.GetOrphanRO(name); err == nil || !apierrors.IsNotFound(err) {
+		return nil, err
+	}
+
+	// labels will be attached by mutator webhook
+	orphan := &longhorn.Orphan{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:            name,
+			OwnerReferences: datastore.GetOwnerReferencesForInstanceManager(im),
+		},
+		Spec: longhorn.OrphanSpec{
+			NodeID:     m.controllerID,
+			Type:       orphanType,
+			DataEngine: dataEngineType,
+			Parameters: map[string]string{
+				longhorn.OrphanInstanceName:    instanceName,
+				longhorn.OrphanInstanceManager: im.Name,
+			},
+		},
+	}
+
+	return m.ds.CreateOrphan(orphan)
+}
+
+func (m *InstanceManagerMonitor) getDeletableOrphans(existOrphans map[string]*longhorn.Orphan, engineProcess instanceProcessMap, replicaProcess instanceProcessMap) map[string]bool {
+	deletableOrphans := make(map[string]bool)
+	for orphanName, orphan := range existOrphans {
+		instanceName := orphan.Spec.Parameters[longhorn.OrphanInstanceName]
+		var instanceProc longhorn.InstanceProcess
+		var instanceExist, instanceCRScheduledBack bool
+		switch orphan.Spec.Type {
+		case longhorn.OrphanTypeEngineInstance:
+			instanceProc, instanceExist = engineProcess[instanceName]
+			isCRScheduledBack, err := m.isEngineScheduledOnNode(instanceName)
path: controller/instance_manager_controller.go
user: login: derekbit
body: might be more clear
```suggestion
			isCRScheduledBack, err := m.isEngineScheduledOnCurrentNode(instanceName)
```
created_at: 2025-04-23T02:19:57Z
author_association: MEMBER
diff_hunk: @@ -2044,6 +2084,268 @@ func (m *InstanceManagerMonitor) StopMonitorWithLock() {
 	m.done = true
 }
 
+func (m *InstanceManagerMonitor) syncOrphans(im *longhorn.InstanceManager, allInstances instanceProcessMap) {
+	engineProcess, replicaProcess := m.categorizeProcesses(allInstances)
+	existOrphans, err := m.listOrphansForEngineAndReplicaInstances(im.Spec.NodeID)
+	if err != nil {
+		m.logger.WithError(err).Errorf("Failed to list orphans on node %s", im.Spec.NodeID)
+		return
+	}
+
+	// exam instances and create orphan CRs
+	existOrphans = m.createOrphanForInstancesAndUpdateCache(existOrphans, im, engineProcess, longhorn.OrphanTypeEngineInstance, m.isEngineOrphaned)
+	existOrphans = m.createOrphanForInstancesAndUpdateCache(existOrphans, im, replicaProcess, longhorn.OrphanTypeReplicaInstance, m.isReplicaOrphaned)
+
path: controller/instance_manager_controller.go
user: login: derekbit
body: Add a comment for
```
	deletableOrphans := m.getDeletableOrphans(existOrphans, engineProcess, replicaProcess)
	m.deleteOrphans(existOrphans, deletableOrphans)
```

It deletes the orphans that are not used anymore.
created_at: 2025-04-23T02:49:41Z
author_association: MEMBER
diff_hunk: @@ -371,6 +572,55 @@ func (oc *OrphanController) updateConditions(orphan *longhorn.Orphan) error {
 	return nil
 }
 
+// If the InstanceState condition status is true, the corresponding instance is still exist in instance manager
+func (oc *OrphanController) updateInstanceStateCondition(orphan *longhorn.Orphan, instanceType longhorn.InstanceType) (err error) {
+	var instanceState longhorn.InstanceState
+	defer func() {
+		if err != nil {
+			return
+		}
+
+		status := longhorn.ConditionStatusTrue
+		if instanceState == longhorn.InstanceStateTerminated {
+			status = longhorn.ConditionStatusFalse
+		}
+
+		orphan.Status.Conditions = types.SetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeInstanceState, status, string(instanceState), "")
+	}()
+
+	instanceName, instanceManager, err := oc.extractOrphanedInstanceInfo(orphan)
+	im, err := oc.ds.GetInstanceManager(instanceManager)
+	if err != nil {
+		if !apierrors.IsNotFound(err) {
+			oc.logger.WithError(err).Infof("No instance manager for node %v for update instance state of orphan instance %v", oc.controllerID, orphan.Name)
+			instanceState = longhorn.InstanceStateTerminated
+			return nil
+		}
+		return err
+	}
+	imc, err := engineapi.NewInstanceManagerClient(im, false)
+	if err != nil {
+		return errors.Wrapf(err, "failed to check node down or missing manager for node %v", orphan.Spec.NodeID)
+	}
+	defer func() {
+		if closeErr := imc.Close(); closeErr != nil {
+			oc.logger.WithError(closeErr).Error("failed to close instance manager client")
+		}
+	}()
+
+	instanceProc, err := imc.InstanceGet(orphan.Spec.DataEngine, instanceName, string(instanceType))
+	switch {
+	case err != nil:
+		return errors.Wrapf(err, "failed to get instance %v", instanceName)
+	case instanceProc == nil:
+		instanceState = longhorn.InstanceStateTerminated
+	default:
+		instanceState = instanceProc.Status.State
+	}
path: controller/orphan_controller.go
user: login: derekbit
body: Thanks. 
created_at: 2025-04-23T02:54:47Z
author_association: MEMBER
diff_hunk: @@ -269,48 +303,200 @@ func (oc *OrphanController) isResponsibleFor(orphan *longhorn.Orphan) bool {
 	return isControllerResponsibleFor(oc.controllerID, oc.ds, orphan.Name, orphan.Spec.NodeID, orphan.Status.OwnerID)
 }
 
-func (oc *OrphanController) cleanupOrphanedData(orphan *longhorn.Orphan) (err error) {
+func (oc *OrphanController) cleanupOrphanedResource(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
 	log := getLoggerForOrphan(oc.logger, orphan)
 
 	defer func() {
 		if err == nil {
 			return
 		}
 
-		err = errors.Wrapf(err, "failed to delete orphan %v data", orphan.Name)
+		err = errors.Wrapf(err, "failed to delete %v orphan %v", orphan.Spec.Type, orphan.Name)
 		orphan.Status.Conditions = types.SetCondition(orphan.Status.Conditions,
 			longhorn.OrphanConditionTypeError, longhorn.ConditionStatusTrue, "", err.Error())
 	}()
 
 	// Make sure if the orphan nodeID and controller ID are same.
 	// If NO, just delete the orphan resource object and don't touch the data.
 	if orphan.Spec.NodeID != oc.controllerID {
-		log.Infof("Orphan nodeID %v is different from controllerID %v, so just delete the orphan resource object",
-			orphan.Name, oc.controllerID)
-		return nil
+		log.WithFields(logrus.Fields{
+			"orphanType": orphan.Spec.Type,
+			"orphanName": orphan.Name,
+			"orphanNode": orphan.Spec.NodeID,
+		}).Infof("Orphan does not belong to this controller. Skipping resource cleanup")
+		return true, nil
 	}
 
-	if types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeDataCleanable).Status !=
-		longhorn.ConditionStatusTrue {
-		log.Infof("Only delete orphan %v resource object and do not delete the orphaned data", orphan.Name)
-		return nil
+	switch orphan.Spec.Type {
+	case longhorn.OrphanTypeEngineInstance:
+		isCleanupComplete, err = oc.cleanupOrphanedEngineInstance(orphan)
+	case longhorn.OrphanTypeReplicaInstance:
+		isCleanupComplete, err = oc.cleanupOrphanedReplicaInstance(orphan)
+	case longhorn.OrphanTypeReplicaData:
+		if types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeDataCleanable).Status !=
+			longhorn.ConditionStatusTrue {
+			log.Infof("Only delete orphan %v resource object and do not delete the orphaned data store", orphan.Name)
+			return true, nil
+		}
+		err = oc.deleteOrphanedReplicaDataStore(orphan)
+		if err == nil || (err != nil && apierrors.IsNotFound(err)) {
+			isCleanupComplete = true
+			err = nil
+		}
+	default:
+		return false, fmt.Errorf("unknown orphan type %v to clean up orphaned resource for %v", orphan.Spec.Type, orphan.Name)
 	}
 
-	switch orphan.Spec.Type {
-	case longhorn.OrphanTypeReplica:
-		err = oc.deleteOrphanedReplica(orphan)
+	return isCleanupComplete, err
+}
+
+func (oc *OrphanController) cleanupOrphanedEngineInstance(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
+	defer func() {
+		if err != nil {
+			err = errors.Wrapf(err, "cannot cleanup orphan engine instance %v", orphan.Name)
+		}
+	}()
+
+	instance, imName, err := oc.extractOrphanedInstanceInfo(orphan)
+	if err != nil {
+		return false, err
+	}
+
+	var spec *longhorn.InstanceSpec
+	if engineCR, err := oc.ds.GetEngineRO(instance); err != nil {
+		if !apierrors.IsNotFound(err) {
+			return false, err
+		}
+		spec = nil
+	} else {
+		spec = &engineCR.Spec.InstanceSpec
+	}
+	return oc.cleanupOrphanedInstance(orphan, instance, imName, longhorn.InstanceManagerTypeEngine, orphan.Spec.DataEngine, spec)
+}
+
+func (oc *OrphanController) cleanupOrphanedReplicaInstance(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
+	defer func() {
+		if err != nil {
+			err = errors.Wrapf(err, "cannot cleanup orphan replica instance %v", orphan.Name)
+		}
+	}()
+
+	instance, imName, err := oc.extractOrphanedInstanceInfo(orphan)
+	if err != nil {
+		return false, err
+	}
+
+	var spec *longhorn.InstanceSpec
+	if replicaCR, err := oc.ds.GetReplicaRO(instance); err != nil {
+		if !apierrors.IsNotFound(err) {
+			return false, err
+		}
+		spec = nil
+	} else {
+		spec = &replicaCR.Spec.InstanceSpec
+	}
+	return oc.cleanupOrphanedInstance(orphan, instance, imName, longhorn.InstanceManagerTypeReplica, orphan.Spec.DataEngine, spec)
+}
+
+func (oc *OrphanController) cleanupOrphanedInstance(orphan *longhorn.Orphan, instance, imName string, imType longhorn.InstanceManagerType, deType longhorn.DataEngineType, instanceCRSpec *longhorn.InstanceSpec) (isCleanupComplete bool, err error) {
+	if instanceCRSpec != nil && instanceCRSpec.NodeID == oc.controllerID {
+		// Instance is scheduled back to current node and is no longer considered orphaned.
+		// Finalize the orphan CR directly without terminating the instance process.
+		return true, nil
+	}
+
+	im, err := oc.ds.GetRunningInstanceManagerByNodeRO(oc.controllerID, deType)
+	if err != nil {
+		oc.logger.WithError(err).Infof("No running instance manager for node %v for deleting orphan instance %v", oc.controllerID, orphan.Name)
+		return true, nil
+	}
+	if im.Name != imName {
+		oc.logger.WithField("instanceManager", imName).Infof("Orphan instance %v is not managed by current running instance manager %v", orphan.Name, im.Name)
+		return true, nil
+	}
+	imc, err := engineapi.NewInstanceManagerClient(im, false)
+	if err != nil {
+		return false, err
+	}
+	defer func() {
+		if closeErr := imc.Close(); closeErr != nil {
+			oc.logger.WithError(closeErr).Error("failed to close instance manager client")
+		}
+	}()
+
+	instanceExist := types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeInstanceExist).Status == longhorn.ConditionStatusTrue
+	if instanceExist {
+		if err = oc.deleteInstance(imc, instance, imType, deType); err != nil {
+			return false, err
+		}
+	}
path: controller/orphan_controller.go
user: login: derekbit
body: Do we really need to check the condition? Can we blindly delete the instance?
created_at: 2025-04-23T03:03:48Z
author_association: MEMBER
diff_hunk: @@ -269,48 +303,200 @@ func (oc *OrphanController) isResponsibleFor(orphan *longhorn.Orphan) bool {
 	return isControllerResponsibleFor(oc.controllerID, oc.ds, orphan.Name, orphan.Spec.NodeID, orphan.Status.OwnerID)
 }
 
-func (oc *OrphanController) cleanupOrphanedData(orphan *longhorn.Orphan) (err error) {
+func (oc *OrphanController) cleanupOrphanedResource(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
 	log := getLoggerForOrphan(oc.logger, orphan)
 
 	defer func() {
 		if err == nil {
 			return
 		}
 
-		err = errors.Wrapf(err, "failed to delete orphan %v data", orphan.Name)
+		err = errors.Wrapf(err, "failed to delete %v orphan %v", orphan.Spec.Type, orphan.Name)
 		orphan.Status.Conditions = types.SetCondition(orphan.Status.Conditions,
 			longhorn.OrphanConditionTypeError, longhorn.ConditionStatusTrue, "", err.Error())
 	}()
 
 	// Make sure if the orphan nodeID and controller ID are same.
 	// If NO, just delete the orphan resource object and don't touch the data.
 	if orphan.Spec.NodeID != oc.controllerID {
-		log.Infof("Orphan nodeID %v is different from controllerID %v, so just delete the orphan resource object",
-			orphan.Name, oc.controllerID)
-		return nil
+		log.WithFields(logrus.Fields{
+			"orphanType": orphan.Spec.Type,
+			"orphanName": orphan.Name,
+			"orphanNode": orphan.Spec.NodeID,
+		}).Infof("Orphan does not belong to this controller. Skipping resource cleanup")
+		return true, nil
 	}
 
-	if types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeDataCleanable).Status !=
-		longhorn.ConditionStatusTrue {
-		log.Infof("Only delete orphan %v resource object and do not delete the orphaned data", orphan.Name)
-		return nil
+	switch orphan.Spec.Type {
+	case longhorn.OrphanTypeEngineInstance:
+		isCleanupComplete, err = oc.cleanupOrphanedEngineInstance(orphan)
+	case longhorn.OrphanTypeReplicaInstance:
+		isCleanupComplete, err = oc.cleanupOrphanedReplicaInstance(orphan)
+	case longhorn.OrphanTypeReplicaData:
+		if types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeDataCleanable).Status !=
+			longhorn.ConditionStatusTrue {
+			log.Infof("Only delete orphan %v resource object and do not delete the orphaned data store", orphan.Name)
+			return true, nil
+		}
+		err = oc.deleteOrphanedReplicaDataStore(orphan)
+		if err == nil || (err != nil && apierrors.IsNotFound(err)) {
+			isCleanupComplete = true
+			err = nil
+		}
+	default:
+		return false, fmt.Errorf("unknown orphan type %v to clean up orphaned resource for %v", orphan.Spec.Type, orphan.Name)
 	}
 
-	switch orphan.Spec.Type {
-	case longhorn.OrphanTypeReplica:
-		err = oc.deleteOrphanedReplica(orphan)
+	return isCleanupComplete, err
+}
+
+func (oc *OrphanController) cleanupOrphanedEngineInstance(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
+	defer func() {
+		if err != nil {
+			err = errors.Wrapf(err, "cannot cleanup orphan engine instance %v", orphan.Name)
+		}
+	}()
+
+	instance, imName, err := oc.extractOrphanedInstanceInfo(orphan)
+	if err != nil {
+		return false, err
+	}
+
+	var spec *longhorn.InstanceSpec
+	if engineCR, err := oc.ds.GetEngineRO(instance); err != nil {
+		if !apierrors.IsNotFound(err) {
+			return false, err
+		}
+		spec = nil
+	} else {
+		spec = &engineCR.Spec.InstanceSpec
+	}
+	return oc.cleanupOrphanedInstance(orphan, instance, imName, longhorn.InstanceManagerTypeEngine, orphan.Spec.DataEngine, spec)
+}
+
+func (oc *OrphanController) cleanupOrphanedReplicaInstance(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
+	defer func() {
+		if err != nil {
+			err = errors.Wrapf(err, "cannot cleanup orphan replica instance %v", orphan.Name)
+		}
+	}()
+
+	instance, imName, err := oc.extractOrphanedInstanceInfo(orphan)
+	if err != nil {
+		return false, err
+	}
+
+	var spec *longhorn.InstanceSpec
+	if replicaCR, err := oc.ds.GetReplicaRO(instance); err != nil {
+		if !apierrors.IsNotFound(err) {
+			return false, err
+		}
+		spec = nil
+	} else {
+		spec = &replicaCR.Spec.InstanceSpec
+	}
+	return oc.cleanupOrphanedInstance(orphan, instance, imName, longhorn.InstanceManagerTypeReplica, orphan.Spec.DataEngine, spec)
+}
+
+func (oc *OrphanController) cleanupOrphanedInstance(orphan *longhorn.Orphan, instance, imName string, imType longhorn.InstanceManagerType, deType longhorn.DataEngineType, instanceCRSpec *longhorn.InstanceSpec) (isCleanupComplete bool, err error) {
+	if instanceCRSpec != nil && instanceCRSpec.NodeID == oc.controllerID {
+		// Instance is scheduled back to current node and is no longer considered orphaned.
+		// Finalize the orphan CR directly without terminating the instance process.
+		return true, nil
+	}
+
+	im, err := oc.ds.GetRunningInstanceManagerByNodeRO(oc.controllerID, deType)
+	if err != nil {
+		oc.logger.WithError(err).Infof("No running instance manager for node %v for deleting orphan instance %v", oc.controllerID, orphan.Name)
+		return true, nil
+	}
+	if im.Name != imName {
+		oc.logger.WithField("instanceManager", imName).Infof("Orphan instance %v is not managed by current running instance manager %v", orphan.Name, im.Name)
+		return true, nil
+	}
path: controller/orphan_controller.go
user: login: derekbit
body: nit: wrap the codes into a helper function like `isCleanupOrphanedInstanceRequired`?
created_at: 2025-04-23T03:04:41Z
author_association: MEMBER
diff_hunk: @@ -269,48 +303,200 @@ func (oc *OrphanController) isResponsibleFor(orphan *longhorn.Orphan) bool {
 	return isControllerResponsibleFor(oc.controllerID, oc.ds, orphan.Name, orphan.Spec.NodeID, orphan.Status.OwnerID)
 }
 
-func (oc *OrphanController) cleanupOrphanedData(orphan *longhorn.Orphan) (err error) {
+func (oc *OrphanController) cleanupOrphanedResource(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
 	log := getLoggerForOrphan(oc.logger, orphan)
 
 	defer func() {
 		if err == nil {
 			return
 		}
 
-		err = errors.Wrapf(err, "failed to delete orphan %v data", orphan.Name)
+		err = errors.Wrapf(err, "failed to delete %v orphan %v", orphan.Spec.Type, orphan.Name)
 		orphan.Status.Conditions = types.SetCondition(orphan.Status.Conditions,
 			longhorn.OrphanConditionTypeError, longhorn.ConditionStatusTrue, "", err.Error())
 	}()
 
 	// Make sure if the orphan nodeID and controller ID are same.
 	// If NO, just delete the orphan resource object and don't touch the data.
 	if orphan.Spec.NodeID != oc.controllerID {
-		log.Infof("Orphan nodeID %v is different from controllerID %v, so just delete the orphan resource object",
-			orphan.Name, oc.controllerID)
-		return nil
+		log.WithFields(logrus.Fields{
+			"orphanType": orphan.Spec.Type,
+			"orphanName": orphan.Name,
+			"orphanNode": orphan.Spec.NodeID,
+		}).Infof("Orphan does not belong to this controller. Skipping resource cleanup")
+		return true, nil
 	}
 
-	if types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeDataCleanable).Status !=
-		longhorn.ConditionStatusTrue {
-		log.Infof("Only delete orphan %v resource object and do not delete the orphaned data", orphan.Name)
-		return nil
+	switch orphan.Spec.Type {
+	case longhorn.OrphanTypeEngineInstance:
+		isCleanupComplete, err = oc.cleanupOrphanedEngineInstance(orphan)
+	case longhorn.OrphanTypeReplicaInstance:
+		isCleanupComplete, err = oc.cleanupOrphanedReplicaInstance(orphan)
+	case longhorn.OrphanTypeReplicaData:
+		if types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeDataCleanable).Status !=
+			longhorn.ConditionStatusTrue {
+			log.Infof("Only delete orphan %v resource object and do not delete the orphaned data store", orphan.Name)
+			return true, nil
+		}
+		err = oc.deleteOrphanedReplicaDataStore(orphan)
+		if err == nil || (err != nil && apierrors.IsNotFound(err)) {
+			isCleanupComplete = true
+			err = nil
+		}
+	default:
+		return false, fmt.Errorf("unknown orphan type %v to clean up orphaned resource for %v", orphan.Spec.Type, orphan.Name)
 	}
 
-	switch orphan.Spec.Type {
-	case longhorn.OrphanTypeReplica:
-		err = oc.deleteOrphanedReplica(orphan)
+	return isCleanupComplete, err
+}
+
+func (oc *OrphanController) cleanupOrphanedEngineInstance(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
+	defer func() {
+		if err != nil {
+			err = errors.Wrapf(err, "cannot cleanup orphan engine instance %v", orphan.Name)
+		}
+	}()
+
+	instance, imName, err := oc.extractOrphanedInstanceInfo(orphan)
+	if err != nil {
+		return false, err
+	}
+
+	var spec *longhorn.InstanceSpec
+	if engineCR, err := oc.ds.GetEngineRO(instance); err != nil {
+		if !apierrors.IsNotFound(err) {
+			return false, err
+		}
+		spec = nil
+	} else {
+		spec = &engineCR.Spec.InstanceSpec
+	}
+	return oc.cleanupOrphanedInstance(orphan, instance, imName, longhorn.InstanceManagerTypeEngine, orphan.Spec.DataEngine, spec)
+}
+
+func (oc *OrphanController) cleanupOrphanedReplicaInstance(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
+	defer func() {
+		if err != nil {
+			err = errors.Wrapf(err, "cannot cleanup orphan replica instance %v", orphan.Name)
+		}
+	}()
+
+	instance, imName, err := oc.extractOrphanedInstanceInfo(orphan)
+	if err != nil {
+		return false, err
+	}
+
+	var spec *longhorn.InstanceSpec
+	if replicaCR, err := oc.ds.GetReplicaRO(instance); err != nil {
+		if !apierrors.IsNotFound(err) {
+			return false, err
+		}
+		spec = nil
+	} else {
+		spec = &replicaCR.Spec.InstanceSpec
+	}
+	return oc.cleanupOrphanedInstance(orphan, instance, imName, longhorn.InstanceManagerTypeReplica, orphan.Spec.DataEngine, spec)
path: controller/orphan_controller.go
user: login: derekbit
body: `orphan` is one of the parameters. Should we need `orphan.Spec.DataEngine` here?
created_at: 2025-04-23T03:06:02Z
author_association: MEMBER
diff_hunk: @@ -269,48 +303,200 @@ func (oc *OrphanController) isResponsibleFor(orphan *longhorn.Orphan) bool {
 	return isControllerResponsibleFor(oc.controllerID, oc.ds, orphan.Name, orphan.Spec.NodeID, orphan.Status.OwnerID)
 }
 
-func (oc *OrphanController) cleanupOrphanedData(orphan *longhorn.Orphan) (err error) {
+func (oc *OrphanController) cleanupOrphanedResource(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
 	log := getLoggerForOrphan(oc.logger, orphan)
 
 	defer func() {
 		if err == nil {
 			return
 		}
 
-		err = errors.Wrapf(err, "failed to delete orphan %v data", orphan.Name)
+		err = errors.Wrapf(err, "failed to delete %v orphan %v", orphan.Spec.Type, orphan.Name)
 		orphan.Status.Conditions = types.SetCondition(orphan.Status.Conditions,
 			longhorn.OrphanConditionTypeError, longhorn.ConditionStatusTrue, "", err.Error())
 	}()
 
 	// Make sure if the orphan nodeID and controller ID are same.
 	// If NO, just delete the orphan resource object and don't touch the data.
 	if orphan.Spec.NodeID != oc.controllerID {
-		log.Infof("Orphan nodeID %v is different from controllerID %v, so just delete the orphan resource object",
-			orphan.Name, oc.controllerID)
-		return nil
+		log.WithFields(logrus.Fields{
+			"orphanType": orphan.Spec.Type,
+			"orphanName": orphan.Name,
+			"orphanNode": orphan.Spec.NodeID,
+		}).Infof("Orphan does not belong to this controller. Skipping resource cleanup")
+		return true, nil
 	}
 
-	if types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeDataCleanable).Status !=
-		longhorn.ConditionStatusTrue {
-		log.Infof("Only delete orphan %v resource object and do not delete the orphaned data", orphan.Name)
-		return nil
+	switch orphan.Spec.Type {
+	case longhorn.OrphanTypeEngineInstance:
+		isCleanupComplete, err = oc.cleanupOrphanedEngineInstance(orphan)
+	case longhorn.OrphanTypeReplicaInstance:
+		isCleanupComplete, err = oc.cleanupOrphanedReplicaInstance(orphan)
+	case longhorn.OrphanTypeReplicaData:
+		if types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeDataCleanable).Status !=
+			longhorn.ConditionStatusTrue {
+			log.Infof("Only delete orphan %v resource object and do not delete the orphaned data store", orphan.Name)
+			return true, nil
+		}
+		err = oc.deleteOrphanedReplicaDataStore(orphan)
+		if err == nil || (err != nil && apierrors.IsNotFound(err)) {
+			isCleanupComplete = true
+			err = nil
+		}
+	default:
+		return false, fmt.Errorf("unknown orphan type %v to clean up orphaned resource for %v", orphan.Spec.Type, orphan.Name)
 	}
 
-	switch orphan.Spec.Type {
-	case longhorn.OrphanTypeReplica:
-		err = oc.deleteOrphanedReplica(orphan)
+	return isCleanupComplete, err
+}
+
+func (oc *OrphanController) cleanupOrphanedEngineInstance(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
+	defer func() {
+		if err != nil {
+			err = errors.Wrapf(err, "cannot cleanup orphan engine instance %v", orphan.Name)
+		}
+	}()
+
+	instance, imName, err := oc.extractOrphanedInstanceInfo(orphan)
+	if err != nil {
+		return false, err
+	}
+
+	var spec *longhorn.InstanceSpec
+	if engineCR, err := oc.ds.GetEngineRO(instance); err != nil {
+		if !apierrors.IsNotFound(err) {
+			return false, err
+		}
+		spec = nil
+	} else {
+		spec = &engineCR.Spec.InstanceSpec
+	}
+	return oc.cleanupOrphanedInstance(orphan, instance, imName, longhorn.InstanceManagerTypeEngine, orphan.Spec.DataEngine, spec)
path: controller/orphan_controller.go
user: login: derekbit
body: `orphan` is one of the parameters. Should we need `orphan.Spec.DataEngine` here?
created_at: 2025-04-23T03:06:07Z
author_association: MEMBER
diff_hunk: @@ -337,7 +523,9 @@ func (oc *OrphanController) DeleteV2ReplicaInstance(diskName, diskUUID, diskDriv
 	logrus.Infof("Deleting SPDK replica instance %v on disk %v on node %v", replicaInstanceName, diskUUID, oc.controllerID)
 
 	defer func() {
-		err = errors.Wrapf(err, "cannot delete v2 replica instance %v", replicaInstanceName)
+		if err != nil {
+			err = errors.Wrapf(err, "cannot delete v2 replica instance %v", replicaInstanceName)
+		}
path: controller/orphan_controller.go
user: login: derekbit
body: `if err != nil {` is not required. If `err` is nil, the result is nil as well.
created_at: 2025-04-23T03:07:10Z
author_association: MEMBER
diff_hunk: @@ -269,48 +303,200 @@ func (oc *OrphanController) isResponsibleFor(orphan *longhorn.Orphan) bool {
 	return isControllerResponsibleFor(oc.controllerID, oc.ds, orphan.Name, orphan.Spec.NodeID, orphan.Status.OwnerID)
 }
 
-func (oc *OrphanController) cleanupOrphanedData(orphan *longhorn.Orphan) (err error) {
+func (oc *OrphanController) cleanupOrphanedResource(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
 	log := getLoggerForOrphan(oc.logger, orphan)
 
 	defer func() {
 		if err == nil {
 			return
 		}
 
-		err = errors.Wrapf(err, "failed to delete orphan %v data", orphan.Name)
+		err = errors.Wrapf(err, "failed to delete %v orphan %v", orphan.Spec.Type, orphan.Name)
 		orphan.Status.Conditions = types.SetCondition(orphan.Status.Conditions,
 			longhorn.OrphanConditionTypeError, longhorn.ConditionStatusTrue, "", err.Error())
 	}()
 
 	// Make sure if the orphan nodeID and controller ID are same.
 	// If NO, just delete the orphan resource object and don't touch the data.
 	if orphan.Spec.NodeID != oc.controllerID {
-		log.Infof("Orphan nodeID %v is different from controllerID %v, so just delete the orphan resource object",
-			orphan.Name, oc.controllerID)
-		return nil
+		log.WithFields(logrus.Fields{
+			"orphanType": orphan.Spec.Type,
+			"orphanName": orphan.Name,
+			"orphanNode": orphan.Spec.NodeID,
+		}).Infof("Orphan does not belong to this controller. Skipping resource cleanup")
+		return true, nil
 	}
 
-	if types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeDataCleanable).Status !=
-		longhorn.ConditionStatusTrue {
-		log.Infof("Only delete orphan %v resource object and do not delete the orphaned data", orphan.Name)
-		return nil
+	switch orphan.Spec.Type {
+	case longhorn.OrphanTypeEngineInstance:
+		isCleanupComplete, err = oc.cleanupOrphanedEngineInstance(orphan)
+	case longhorn.OrphanTypeReplicaInstance:
+		isCleanupComplete, err = oc.cleanupOrphanedReplicaInstance(orphan)
+	case longhorn.OrphanTypeReplicaData:
+		if types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeDataCleanable).Status !=
+			longhorn.ConditionStatusTrue {
+			log.Infof("Only delete orphan %v resource object and do not delete the orphaned data store", orphan.Name)
+			return true, nil
+		}
+		err = oc.deleteOrphanedReplicaDataStore(orphan)
+		if err == nil || (err != nil && apierrors.IsNotFound(err)) {
+			isCleanupComplete = true
+			err = nil
+		}
+	default:
+		return false, fmt.Errorf("unknown orphan type %v to clean up orphaned resource for %v", orphan.Spec.Type, orphan.Name)
 	}
 
-	switch orphan.Spec.Type {
-	case longhorn.OrphanTypeReplica:
-		err = oc.deleteOrphanedReplica(orphan)
+	return isCleanupComplete, err
+}
+
+func (oc *OrphanController) cleanupOrphanedEngineInstance(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
+	defer func() {
+		if err != nil {
+			err = errors.Wrapf(err, "cannot cleanup orphan engine instance %v", orphan.Name)
+		}
+	}()
+
+	instance, imName, err := oc.extractOrphanedInstanceInfo(orphan)
+	if err != nil {
+		return false, err
+	}
+
+	var spec *longhorn.InstanceSpec
+	if engineCR, err := oc.ds.GetEngineRO(instance); err != nil {
+		if !apierrors.IsNotFound(err) {
+			return false, err
path: controller/orphan_controller.go
user: login: COLDTURNIP
body: Yes. When the engine CR is absent later in `cleanupOrphanedInstance`, it skip the CR node location examination and continue on instance deletion.
created_at: 2025-04-23T03:17:06Z
author_association: MEMBER
diff_hunk: @@ -269,48 +303,200 @@ func (oc *OrphanController) isResponsibleFor(orphan *longhorn.Orphan) bool {
 	return isControllerResponsibleFor(oc.controllerID, oc.ds, orphan.Name, orphan.Spec.NodeID, orphan.Status.OwnerID)
 }
 
-func (oc *OrphanController) cleanupOrphanedData(orphan *longhorn.Orphan) (err error) {
+func (oc *OrphanController) cleanupOrphanedResource(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
 	log := getLoggerForOrphan(oc.logger, orphan)
 
 	defer func() {
 		if err == nil {
 			return
 		}
 
-		err = errors.Wrapf(err, "failed to delete orphan %v data", orphan.Name)
+		err = errors.Wrapf(err, "failed to delete %v orphan %v", orphan.Spec.Type, orphan.Name)
 		orphan.Status.Conditions = types.SetCondition(orphan.Status.Conditions,
 			longhorn.OrphanConditionTypeError, longhorn.ConditionStatusTrue, "", err.Error())
 	}()
 
 	// Make sure if the orphan nodeID and controller ID are same.
 	// If NO, just delete the orphan resource object and don't touch the data.
 	if orphan.Spec.NodeID != oc.controllerID {
-		log.Infof("Orphan nodeID %v is different from controllerID %v, so just delete the orphan resource object",
-			orphan.Name, oc.controllerID)
-		return nil
+		log.WithFields(logrus.Fields{
+			"orphanType": orphan.Spec.Type,
+			"orphanName": orphan.Name,
+			"orphanNode": orphan.Spec.NodeID,
+		}).Infof("Orphan does not belong to this controller. Skipping resource cleanup")
+		return true, nil
 	}
 
-	if types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeDataCleanable).Status !=
-		longhorn.ConditionStatusTrue {
-		log.Infof("Only delete orphan %v resource object and do not delete the orphaned data", orphan.Name)
-		return nil
+	switch orphan.Spec.Type {
+	case longhorn.OrphanTypeEngineInstance:
+		isCleanupComplete, err = oc.cleanupOrphanedEngineInstance(orphan)
+	case longhorn.OrphanTypeReplicaInstance:
+		isCleanupComplete, err = oc.cleanupOrphanedReplicaInstance(orphan)
+	case longhorn.OrphanTypeReplicaData:
+		if types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeDataCleanable).Status !=
+			longhorn.ConditionStatusTrue {
+			log.Infof("Only delete orphan %v resource object and do not delete the orphaned data store", orphan.Name)
+			return true, nil
+		}
+		err = oc.deleteOrphanedReplicaDataStore(orphan)
+		if err == nil || (err != nil && apierrors.IsNotFound(err)) {
+			isCleanupComplete = true
+			err = nil
+		}
+	default:
+		return false, fmt.Errorf("unknown orphan type %v to clean up orphaned resource for %v", orphan.Spec.Type, orphan.Name)
 	}
 
-	switch orphan.Spec.Type {
-	case longhorn.OrphanTypeReplica:
-		err = oc.deleteOrphanedReplica(orphan)
+	return isCleanupComplete, err
+}
+
+func (oc *OrphanController) cleanupOrphanedEngineInstance(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
+	defer func() {
+		if err != nil {
+			err = errors.Wrapf(err, "cannot cleanup orphan engine instance %v", orphan.Name)
+		}
+	}()
+
+	instance, imName, err := oc.extractOrphanedInstanceInfo(orphan)
+	if err != nil {
+		return false, err
+	}
+
+	var spec *longhorn.InstanceSpec
+	if engineCR, err := oc.ds.GetEngineRO(instance); err != nil {
+		if !apierrors.IsNotFound(err) {
+			return false, err
+		}
+		spec = nil
+	} else {
+		spec = &engineCR.Spec.InstanceSpec
+	}
+	return oc.cleanupOrphanedInstance(orphan, instance, imName, longhorn.InstanceManagerTypeEngine, orphan.Spec.DataEngine, spec)
+}
+
+func (oc *OrphanController) cleanupOrphanedReplicaInstance(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
+	defer func() {
+		if err != nil {
+			err = errors.Wrapf(err, "cannot cleanup orphan replica instance %v", orphan.Name)
+		}
+	}()
+
+	instance, imName, err := oc.extractOrphanedInstanceInfo(orphan)
+	if err != nil {
+		return false, err
+	}
+
+	var spec *longhorn.InstanceSpec
+	if replicaCR, err := oc.ds.GetReplicaRO(instance); err != nil {
+		if !apierrors.IsNotFound(err) {
+			return false, err
+		}
+		spec = nil
+	} else {
+		spec = &replicaCR.Spec.InstanceSpec
+	}
+	return oc.cleanupOrphanedInstance(orphan, instance, imName, longhorn.InstanceManagerTypeReplica, orphan.Spec.DataEngine, spec)
+}
+
+func (oc *OrphanController) cleanupOrphanedInstance(orphan *longhorn.Orphan, instance, imName string, imType longhorn.InstanceManagerType, deType longhorn.DataEngineType, instanceCRSpec *longhorn.InstanceSpec) (isCleanupComplete bool, err error) {
+	if instanceCRSpec != nil && instanceCRSpec.NodeID == oc.controllerID {
+		// Instance is scheduled back to current node and is no longer considered orphaned.
+		// Finalize the orphan CR directly without terminating the instance process.
+		return true, nil
+	}
+
+	im, err := oc.ds.GetRunningInstanceManagerByNodeRO(oc.controllerID, deType)
+	if err != nil {
+		oc.logger.WithError(err).Infof("No running instance manager for node %v for deleting orphan instance %v", oc.controllerID, orphan.Name)
+		return true, nil
+	}
+	if im.Name != imName {
+		oc.logger.WithField("instanceManager", imName).Infof("Orphan instance %v is not managed by current running instance manager %v", orphan.Name, im.Name)
+		return true, nil
+	}
path: controller/orphan_controller.go
user: login: COLDTURNIP
body: im is needed to create instance manager client later. Will extract to something like `getRunningInstanceManagerClientForOrphan`, and skip deletion if there's no available client without error.
created_at: 2025-04-23T03:41:15Z
author_association: MEMBER
diff_hunk: @@ -269,48 +303,200 @@ func (oc *OrphanController) isResponsibleFor(orphan *longhorn.Orphan) bool {
 	return isControllerResponsibleFor(oc.controllerID, oc.ds, orphan.Name, orphan.Spec.NodeID, orphan.Status.OwnerID)
 }
 
-func (oc *OrphanController) cleanupOrphanedData(orphan *longhorn.Orphan) (err error) {
+func (oc *OrphanController) cleanupOrphanedResource(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
 	log := getLoggerForOrphan(oc.logger, orphan)
 
 	defer func() {
 		if err == nil {
 			return
 		}
 
-		err = errors.Wrapf(err, "failed to delete orphan %v data", orphan.Name)
+		err = errors.Wrapf(err, "failed to delete %v orphan %v", orphan.Spec.Type, orphan.Name)
 		orphan.Status.Conditions = types.SetCondition(orphan.Status.Conditions,
 			longhorn.OrphanConditionTypeError, longhorn.ConditionStatusTrue, "", err.Error())
 	}()
 
 	// Make sure if the orphan nodeID and controller ID are same.
 	// If NO, just delete the orphan resource object and don't touch the data.
 	if orphan.Spec.NodeID != oc.controllerID {
-		log.Infof("Orphan nodeID %v is different from controllerID %v, so just delete the orphan resource object",
-			orphan.Name, oc.controllerID)
-		return nil
+		log.WithFields(logrus.Fields{
+			"orphanType": orphan.Spec.Type,
+			"orphanName": orphan.Name,
+			"orphanNode": orphan.Spec.NodeID,
+		}).Infof("Orphan does not belong to this controller. Skipping resource cleanup")
+		return true, nil
 	}
 
-	if types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeDataCleanable).Status !=
-		longhorn.ConditionStatusTrue {
-		log.Infof("Only delete orphan %v resource object and do not delete the orphaned data", orphan.Name)
-		return nil
+	switch orphan.Spec.Type {
+	case longhorn.OrphanTypeEngineInstance:
+		isCleanupComplete, err = oc.cleanupOrphanedEngineInstance(orphan)
+	case longhorn.OrphanTypeReplicaInstance:
+		isCleanupComplete, err = oc.cleanupOrphanedReplicaInstance(orphan)
+	case longhorn.OrphanTypeReplicaData:
+		if types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeDataCleanable).Status !=
+			longhorn.ConditionStatusTrue {
+			log.Infof("Only delete orphan %v resource object and do not delete the orphaned data store", orphan.Name)
+			return true, nil
+		}
+		err = oc.deleteOrphanedReplicaDataStore(orphan)
+		if err == nil || (err != nil && apierrors.IsNotFound(err)) {
+			isCleanupComplete = true
+			err = nil
+		}
+	default:
+		return false, fmt.Errorf("unknown orphan type %v to clean up orphaned resource for %v", orphan.Spec.Type, orphan.Name)
 	}
 
-	switch orphan.Spec.Type {
-	case longhorn.OrphanTypeReplica:
-		err = oc.deleteOrphanedReplica(orphan)
+	return isCleanupComplete, err
+}
+
+func (oc *OrphanController) cleanupOrphanedEngineInstance(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
+	defer func() {
+		if err != nil {
+			err = errors.Wrapf(err, "cannot cleanup orphan engine instance %v", orphan.Name)
+		}
+	}()
+
+	instance, imName, err := oc.extractOrphanedInstanceInfo(orphan)
+	if err != nil {
+		return false, err
+	}
+
+	var spec *longhorn.InstanceSpec
+	if engineCR, err := oc.ds.GetEngineRO(instance); err != nil {
+		if !apierrors.IsNotFound(err) {
+			return false, err
+		}
+		spec = nil
+	} else {
+		spec = &engineCR.Spec.InstanceSpec
+	}
+	return oc.cleanupOrphanedInstance(orphan, instance, imName, longhorn.InstanceManagerTypeEngine, orphan.Spec.DataEngine, spec)
+}
+
+func (oc *OrphanController) cleanupOrphanedReplicaInstance(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
+	defer func() {
+		if err != nil {
+			err = errors.Wrapf(err, "cannot cleanup orphan replica instance %v", orphan.Name)
+		}
+	}()
+
+	instance, imName, err := oc.extractOrphanedInstanceInfo(orphan)
+	if err != nil {
+		return false, err
+	}
+
+	var spec *longhorn.InstanceSpec
+	if replicaCR, err := oc.ds.GetReplicaRO(instance); err != nil {
+		if !apierrors.IsNotFound(err) {
+			return false, err
+		}
+		spec = nil
+	} else {
+		spec = &replicaCR.Spec.InstanceSpec
+	}
+	return oc.cleanupOrphanedInstance(orphan, instance, imName, longhorn.InstanceManagerTypeReplica, orphan.Spec.DataEngine, spec)
+}
+
+func (oc *OrphanController) cleanupOrphanedInstance(orphan *longhorn.Orphan, instance, imName string, imType longhorn.InstanceManagerType, deType longhorn.DataEngineType, instanceCRSpec *longhorn.InstanceSpec) (isCleanupComplete bool, err error) {
+	if instanceCRSpec != nil && instanceCRSpec.NodeID == oc.controllerID {
+		// Instance is scheduled back to current node and is no longer considered orphaned.
+		// Finalize the orphan CR directly without terminating the instance process.
+		return true, nil
+	}
+
+	im, err := oc.ds.GetRunningInstanceManagerByNodeRO(oc.controllerID, deType)
+	if err != nil {
+		oc.logger.WithError(err).Infof("No running instance manager for node %v for deleting orphan instance %v", oc.controllerID, orphan.Name)
+		return true, nil
+	}
+	if im.Name != imName {
+		oc.logger.WithField("instanceManager", imName).Infof("Orphan instance %v is not managed by current running instance manager %v", orphan.Name, im.Name)
+		return true, nil
+	}
path: controller/orphan_controller.go
user: login: derekbit
body: Okay. Not easy to abstract this part. Let's keep it as it is. Thank you.
created_at: 2025-04-23T04:07:18Z
author_association: MEMBER
diff_hunk: @@ -2044,6 +2084,268 @@ func (m *InstanceManagerMonitor) StopMonitorWithLock() {
 	m.done = true
 }
 
+func (m *InstanceManagerMonitor) syncOrphans(im *longhorn.InstanceManager, allInstances instanceProcessMap) {
+	engineProcess, replicaProcess := m.categorizeProcesses(allInstances)
+	existOrphans, err := m.listOrphansForEngineAndReplicaInstances(im.Spec.NodeID)
+	if err != nil {
+		m.logger.WithError(err).Errorf("Failed to list orphans on node %s", im.Spec.NodeID)
+		return
+	}
+
+	// exam instances and create orphan CRs
+	existOrphans = m.createOrphanForInstancesAndUpdateCache(existOrphans, im, engineProcess, longhorn.OrphanTypeEngineInstance, m.isEngineOrphaned)
+	existOrphans = m.createOrphanForInstancesAndUpdateCache(existOrphans, im, replicaProcess, longhorn.OrphanTypeReplicaInstance, m.isReplicaOrphaned)
+
path: controller/instance_manager_controller.go
user: login: COLDTURNIP
body: Refactor `getDeletableOrphans` to `isOrphanDeletable` to simplify `deleteOrphans`
created_at: 2025-04-23T04:52:20Z
author_association: MEMBER
diff_hunk: @@ -85,3 +101,49 @@ func upgradeVolumes(namespace string, lhClient *lhclientset.Clientset, resourceM
 
 	return nil
 }
+
+func upgradeSettings(namespace string, lhClient *lhclientset.Clientset, resourceMaps map[string]interface{}) (err error) {
+	defer func() {
+		err = errors.Wrapf(err, upgradeLogPrefix+"upgrade settings failed")
+	}()
+
+	settingsMap, err := upgradeutil.ListAndUpdateSettingsInProvidedCache(namespace, lhClient, resourceMaps)
+	if err != nil {
+		if apierrors.IsNotFound(err) {
+			return nil
+		}
+		return errors.Wrapf(err, "failed to list all existing Longhorn settings during the settings upgrade")
+	}
+
+	if err := updateOrphanResourceAutoDeletionSetting(settingsMap); err != nil {
+		return errors.Wrapf(err, "failed to update orphan resource auto-deletion settings")
+	}
+
+	return nil
+}
+
+func updateOrphanResourceAutoDeletionSetting(settingsMap map[string]*v1beta2.Setting) (err error) {
+	oldSetting, exist := settingsMap[string(types.SettingNameOrphanAutoDeletion)]
+	if !exist {
+		return nil
+	}
+	newSetting, exist := settingsMap[string(types.SettingNameOrphanResourceAutoDeletion)]
+	if !exist {
+		newSetting = &v1beta2.Setting{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: string(types.SettingNameOrphanResourceAutoDeletion),
+			},
+		}
+		settingsMap[string(types.SettingNameOrphanResourceAutoDeletion)] = newSetting
+	}
+	resourceTypes, _ := types.UnmarshalOrphanResourceTypes(newSetting.Value)
path: upgrade/v18xto190/upgrade.go
user: login: derekbit
body: returned `err` should be checked.
created_at: 2025-04-23T09:26:05Z
author_association: MEMBER
diff_hunk: @@ -988,6 +1009,16 @@ func (imc *InstanceManagerController) deleteInstanceManagerPDB(im *longhorn.Inst
 	return nil
 }
 
+func (imc *InstanceManagerController) syncOrphan(im *longhorn.InstanceManager) error {
path: controller/instance_manager_controller.go
user: login: derekbit
body: 
```suggestion
func (imc *InstanceManagerController) syncOrphans(im *longhorn.InstanceManager) error {
```
created_at: 2025-04-23T10:10:40Z
author_association: MEMBER
diff_hunk: @@ -1055,14 +1058,31 @@ var (
 	SettingDefinitionOrphanAutoDeletion = SettingDefinition{
 		DisplayName: "Orphan Auto-Deletion",
 		Description: "This setting allows Longhorn to delete the orphan resource and its corresponding orphaned data automatically. \n\n" +
-			"Orphan resources on down or unknown nodes will not be cleaned up automatically. \n\n",
+			"Orphan resources on down or unknown nodes will not be cleaned up automatically. \n\n" +
+			fmt.Sprintf("Deprecated: enable \"%s\" in %s instead. \n\n", OrphanResourceTypeReplicaData, SettingNameOrphanResourceAutoDeletion),
 		Category: SettingCategoryOrphan,
 		Type:     SettingTypeBool,
 		Required: true,
-		ReadOnly: false,
+		ReadOnly: true,
 		Default:  "false",
 	}
 
+	SettingDefinitionOrphanResourceAutoDeletion = SettingDefinition{
+		DisplayName: "Orphan Resource Auto-Deletion",
+		Description: "This setting allows Longhorn to delete the orphan resource and its corresponding orphaned resource automatically. \n\n" +
path: types/setting.go
user: login: sushant-suse
body: ```suggestion
		Description: "This setting allows Longhorn to automatically delete orphan resources and their corresponding orphaned resources. \n\n" +
```
created_at: 2025-04-23T12:05:04Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1055,14 +1058,31 @@ var (
 	SettingDefinitionOrphanAutoDeletion = SettingDefinition{
 		DisplayName: "Orphan Auto-Deletion",
 		Description: "This setting allows Longhorn to delete the orphan resource and its corresponding orphaned data automatically. \n\n" +
-			"Orphan resources on down or unknown nodes will not be cleaned up automatically. \n\n",
+			"Orphan resources on down or unknown nodes will not be cleaned up automatically. \n\n" +
+			fmt.Sprintf("Deprecated: enable \"%s\" in %s instead. \n\n", OrphanResourceTypeReplicaData, SettingNameOrphanResourceAutoDeletion),
 		Category: SettingCategoryOrphan,
 		Type:     SettingTypeBool,
 		Required: true,
-		ReadOnly: false,
+		ReadOnly: true,
 		Default:  "false",
 	}
 
+	SettingDefinitionOrphanResourceAutoDeletion = SettingDefinition{
+		DisplayName: "Orphan Resource Auto-Deletion",
+		Description: "This setting allows Longhorn to delete the orphan resource and its corresponding orphaned resource automatically. \n\n" +
+			"Orphan resources on down or unknown nodes will not be cleaned up automatically. \n\n" +
path: types/setting.go
user: login: sushant-suse
body: ```suggestion
			"Orphan resources located on nodes that are in down or unknown state will not be cleaned up automatically. \n\n" +
```
created_at: 2025-04-23T12:07:33Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1055,14 +1058,31 @@ var (
 	SettingDefinitionOrphanAutoDeletion = SettingDefinition{
 		DisplayName: "Orphan Auto-Deletion",
 		Description: "This setting allows Longhorn to delete the orphan resource and its corresponding orphaned data automatically. \n\n" +
-			"Orphan resources on down or unknown nodes will not be cleaned up automatically. \n\n",
+			"Orphan resources on down or unknown nodes will not be cleaned up automatically. \n\n" +
+			fmt.Sprintf("Deprecated: enable \"%s\" in %s instead. \n\n", OrphanResourceTypeReplicaData, SettingNameOrphanResourceAutoDeletion),
 		Category: SettingCategoryOrphan,
 		Type:     SettingTypeBool,
 		Required: true,
-		ReadOnly: false,
+		ReadOnly: true,
 		Default:  "false",
 	}
 
+	SettingDefinitionOrphanResourceAutoDeletion = SettingDefinition{
+		DisplayName: "Orphan Resource Auto-Deletion",
+		Description: "This setting allows Longhorn to delete the orphan resource and its corresponding orphaned resource automatically. \n\n" +
+			"Orphan resources on down or unknown nodes will not be cleaned up automatically. \n\n" +
+			"List the enabled resource types in a semicolon-separate list. \n\n" +
path: types/setting.go
user: login: sushant-suse
body: ```suggestion
			"List the enabled resource types in a semicolon-separated list. \n\n" +
```
created_at: 2025-04-23T12:12:20Z
author_association: CONTRIBUTOR
diff_hunk: @@ -2044,6 +2087,265 @@ func (m *InstanceManagerMonitor) StopMonitorWithLock() {
 	m.done = true
 }
 
+func (m *InstanceManagerMonitor) syncOrphans(im *longhorn.InstanceManager, instanceProcess map[string]longhorn.InstanceProcess) {
+	engineProcess := map[string]longhorn.InstanceProcess{}
+	replicaProcess := map[string]longhorn.InstanceProcess{}
+	for name, process := range instanceProcess {
+		switch process.Status.Type {
+		case longhorn.InstanceTypeEngine:
+			engineProcess[name] = process
+		case longhorn.InstanceTypeReplica:
+			replicaProcess[name] = process
+		}
+	}
+
+	existOrphans, err := m.getNodeRuntimeOrphanMap()
+	if err != nil {
+		m.logger.WithError(err).Errorf("Failed to list orphans on node %s", m.controllerID)
+		return
+	}
+
+	// create orphan CR for orphaned instances
+	for _, engineInstance := range engineProcess {
+		if isOrphaned, err := m.isEngineOrphaned(engineInstance.Spec.Name); err != nil {
+			m.logger.WithError(err).Errorf("Failed to check orphan for engine instance %v", engineInstance.Spec.Name)
+		} else if isOrphaned {
+			m.logger.Infof("Engine instance %v is orphaned", engineInstance.Spec.Name)
+			err := m.createOrphan(im, engineInstance.Spec.Name, longhorn.OrphanTypeEngineInstance, engineInstance.Spec.DataEngine)
+			if err != nil {
+				m.logger.WithError(err).Errorf("Failed to create orphan for engine instance %v", engineInstance.Spec.Name)
+			}
+		}
+	}
+	for _, replicaInstance := range replicaProcess {
+		if isOrphaned, err := m.isReplicaOrphaned(replicaInstance.Spec.Name); err != nil {
+			m.logger.WithError(err).Errorf("Failed to check orphan for replica instance %v", replicaInstance.Spec.Name)
+		} else if isOrphaned {
+			m.logger.Infof("Replica instance %v is orphaned", replicaInstance.Spec.Name)
+			err := m.createOrphan(im, replicaInstance.Spec.Name, longhorn.OrphanTypeReplicaInstance, replicaInstance.Spec.DataEngine)
+			if err != nil {
+				m.logger.WithError(err).Errorf("Failed to create orphan for replica instance %v", replicaInstance.Spec.Name)
+			}
+		}
+	}
path: controller/instance_manager_controller.go
user: login: COLDTURNIP
body: It is now `createOrphanForInstances`
created_at: 2025-04-23T13:43:34Z
author_association: MEMBER
diff_hunk: @@ -269,48 +303,205 @@ func (oc *OrphanController) isResponsibleFor(orphan *longhorn.Orphan) bool {
 	return isControllerResponsibleFor(oc.controllerID, oc.ds, orphan.Name, orphan.Spec.NodeID, orphan.Status.OwnerID)
 }
 
-func (oc *OrphanController) cleanupOrphanedData(orphan *longhorn.Orphan) (err error) {
+func (oc *OrphanController) cleanupOrphanedResource(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
 	log := getLoggerForOrphan(oc.logger, orphan)
 
 	defer func() {
 		if err == nil {
 			return
 		}
 
-		err = errors.Wrapf(err, "failed to delete orphan %v data", orphan.Name)
+		err = errors.Wrapf(err, "failed to delete %v orphan %v", orphan.Spec.Type, orphan.Name)
 		orphan.Status.Conditions = types.SetCondition(orphan.Status.Conditions,
 			longhorn.OrphanConditionTypeError, longhorn.ConditionStatusTrue, "", err.Error())
 	}()
 
 	// Make sure if the orphan nodeID and controller ID are same.
 	// If NO, just delete the orphan resource object and don't touch the data.
 	if orphan.Spec.NodeID != oc.controllerID {
-		log.Infof("Orphan nodeID %v is different from controllerID %v, so just delete the orphan resource object",
-			orphan.Name, oc.controllerID)
-		return nil
+		log.WithFields(logrus.Fields{
+			"orphanType": orphan.Spec.Type,
+			"orphanName": orphan.Name,
+			"orphanNode": orphan.Spec.NodeID,
+		}).Infof("Orphan does not belong to this controller. Skipping resource cleanup")
+		return true, nil
 	}
 
-	if types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeDataCleanable).Status !=
-		longhorn.ConditionStatusTrue {
-		log.Infof("Only delete orphan %v resource object and do not delete the orphaned data", orphan.Name)
-		return nil
+	switch orphan.Spec.Type {
+	case longhorn.OrphanTypeEngineInstance:
+		isCleanupComplete, err = oc.cleanupOrphanedEngineInstance(orphan)
+	case longhorn.OrphanTypeReplicaInstance:
+		isCleanupComplete, err = oc.cleanupOrphanedReplicaInstance(orphan)
+	case longhorn.OrphanTypeReplicaData:
+		if types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeDataCleanable).Status !=
+			longhorn.ConditionStatusTrue {
+			log.Infof("Only delete orphan %v resource object and do not delete the orphaned data store", orphan.Name)
+			return true, nil
+		}
+		err = oc.deleteOrphanedReplicaDataStore(orphan)
+		if err == nil || (err != nil && apierrors.IsNotFound(err)) {
+			isCleanupComplete = true
+			err = nil
+		}
+	default:
+		return false, fmt.Errorf("unknown orphan type %v to clean up orphaned resource for %v", orphan.Spec.Type, orphan.Name)
 	}
 
-	switch orphan.Spec.Type {
-	case longhorn.OrphanTypeReplica:
-		err = oc.deleteOrphanedReplica(orphan)
+	return isCleanupComplete, err
+}
+
+func (oc *OrphanController) cleanupOrphanedEngineInstance(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
+	defer func() {
+		if err != nil {
+			err = errors.Wrapf(err, "cannot cleanup orphan engine instance %v", orphan.Name)
+		}
+	}()
+
+	instance, imName, err := oc.extractOrphanedInstanceInfo(orphan)
+	if err != nil {
+		return false, err
+	}
+
+	var spec *longhorn.InstanceSpec
+	if engineCR, err := oc.ds.GetEngineRO(instance); err != nil {
+		if !apierrors.IsNotFound(err) {
+			return false, err
+		}
+		spec = nil
+	} else {
+		spec = &engineCR.Spec.InstanceSpec
+	}
+	return oc.cleanupOrphanedInstance(orphan, instance, imName, longhorn.InstanceManagerTypeEngine, spec)
+}
+
+func (oc *OrphanController) cleanupOrphanedReplicaInstance(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
+	defer func() {
+		if err != nil {
+			err = errors.Wrapf(err, "cannot cleanup orphan replica instance %v", orphan.Name)
+		}
+	}()
+
+	instance, imName, err := oc.extractOrphanedInstanceInfo(orphan)
+	if err != nil {
+		return false, err
+	}
+
+	var spec *longhorn.InstanceSpec
+	if replicaCR, err := oc.ds.GetReplicaRO(instance); err != nil {
+		if !apierrors.IsNotFound(err) {
+			return false, err
+		}
+		spec = nil
+	} else {
+		spec = &replicaCR.Spec.InstanceSpec
+	}
+	return oc.cleanupOrphanedInstance(orphan, instance, imName, longhorn.InstanceManagerTypeReplica, spec)
+}
+
+func (oc *OrphanController) extractOrphanedInstanceInfo(orphan *longhorn.Orphan) (name, instanceManager string, err error) {
+	name, ok := orphan.Spec.Parameters[longhorn.OrphanInstanceName]
+	if !ok {
+		return "", "", fmt.Errorf("failed to get instance name for instance orphan %v", orphan.Name)
+	}
+
+	instanceManager, ok = orphan.Spec.Parameters[longhorn.OrphanInstanceManager]
+	if !ok {
+		return "", "", fmt.Errorf("failed to get instance manager for instance orphan %v", orphan.Name)
+	}
+
+	switch orphan.Spec.DataEngine {
+	case longhorn.DataEngineTypeV1, longhorn.DataEngineTypeV2:
+		// supported data engine type
 	default:
-		err = fmt.Errorf("unknown orphan type %v", orphan.Spec.Type)
+		return "", "", fmt.Errorf("unknown data engine type %v for instance orphan %v", orphan.Spec.DataEngine, orphan.Name)
 	}
 
-	if err == nil || (err != nil && apierrors.IsNotFound(err)) {
-		return nil
+	return name, instanceManager, nil
+}
+
+func (oc *OrphanController) cleanupOrphanedInstance(orphan *longhorn.Orphan, instance, imName string, imType longhorn.InstanceManagerType, instanceCRSpec *longhorn.InstanceSpec) (isCleanupComplete bool, err error) {
+	if instanceCRSpec != nil && instanceCRSpec.NodeID == orphan.Spec.NodeID {
+		oc.logger.Infof("Orphan instance %v is scheduled back to current node %v. Skip cleaning up the instance resource and finalize the orphan CR.", instance, orphan.Spec.NodeID)
+		return true, nil
 	}
 
-	return err
+	imc, err := oc.getRunningInstanceManagerClientForOrphan(orphan, imName)
+	if err != nil {
+		return false, errors.Wrapf(err, "cannot get running instance manager client for orphan %v", orphan.Name)
+	} else if imc == nil {
+		oc.logger.WithField("orphanInstanceNode", orphan.Spec.NodeID).Warnf("No running instance manager for deleting orphan instance %v", orphan.Name)
+		return true, nil
+	}
+	defer func() {
+		if closeErr := imc.Close(); closeErr != nil {
+			oc.logger.WithError(closeErr).Error("Failed to close instance manager client")
+		}
+	}()
+
+	if err := oc.deleteInstance(imc, instance, imType, orphan.Spec.DataEngine); err != nil {
+		return false, err
+	}
+	isCleanupComplete, cleanupErr := oc.confirmOrphanInstanceCleanup(imc, instance, imType, orphan.Spec.DataEngine)
+	if cleanupErr == nil && !isCleanupComplete {
+		oc.logger.Infof("Orphan instance %v cleanup in progress", instance)
+	}
+	return isCleanupComplete, cleanupErr
 }
 
-func (oc *OrphanController) deleteOrphanedReplica(orphan *longhorn.Orphan) error {
+func (oc *OrphanController) getRunningInstanceManagerClientForOrphan(orphan *longhorn.Orphan, imName string) (*engineapi.InstanceManagerClient, error) {
+	im, err := oc.ds.GetRunningInstanceManagerByNodeRO(orphan.Spec.NodeID, orphan.Spec.DataEngine)
+	if err != nil {
+		oc.logger.WithError(err).Warnf("No running instance manager for node %v for deleting orphan instance %v", oc.controllerID, orphan.Name)
+		return nil, nil
+	}
+	if im.Name != imName {
+		oc.logger.WithField("instanceManager", imName).Warnf("Orphan instance %v is not managed by current running instance manager %v", orphan.Name, im.Name)
+		return nil, nil
+	}
+	return engineapi.NewInstanceManagerClient(im, false)
+}
+
+func (oc *OrphanController) deleteInstance(imc *engineapi.InstanceManagerClient, instanceName string, instanceKind longhorn.InstanceManagerType, engineType longhorn.DataEngineType) (err error) {
+	// There is a delay between deletion initiation and state/InstanceManager update,
+	// this function may be called multiple times before given instance exits.
+
+	oc.logger.Infof("Orphan controller deleting instance %v", instanceName)
+
+	defer func() {
+		err = errors.Wrapf(err, "cannot delete instance %v", instanceName)
path: controller/orphan_controller.go
user: login: derekbit
body: 
```suggestion
		err = errors.Wrapf(err, "failed to delete instance %v", instanceName)
```
created_at: 2025-04-23T14:58:36Z
author_association: MEMBER
diff_hunk: @@ -269,48 +303,205 @@ func (oc *OrphanController) isResponsibleFor(orphan *longhorn.Orphan) bool {
 	return isControllerResponsibleFor(oc.controllerID, oc.ds, orphan.Name, orphan.Spec.NodeID, orphan.Status.OwnerID)
 }
 
-func (oc *OrphanController) cleanupOrphanedData(orphan *longhorn.Orphan) (err error) {
+func (oc *OrphanController) cleanupOrphanedResource(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
 	log := getLoggerForOrphan(oc.logger, orphan)
 
 	defer func() {
 		if err == nil {
 			return
 		}
 
-		err = errors.Wrapf(err, "failed to delete orphan %v data", orphan.Name)
+		err = errors.Wrapf(err, "failed to delete %v orphan %v", orphan.Spec.Type, orphan.Name)
 		orphan.Status.Conditions = types.SetCondition(orphan.Status.Conditions,
 			longhorn.OrphanConditionTypeError, longhorn.ConditionStatusTrue, "", err.Error())
 	}()
 
 	// Make sure if the orphan nodeID and controller ID are same.
 	// If NO, just delete the orphan resource object and don't touch the data.
 	if orphan.Spec.NodeID != oc.controllerID {
-		log.Infof("Orphan nodeID %v is different from controllerID %v, so just delete the orphan resource object",
-			orphan.Name, oc.controllerID)
-		return nil
+		log.WithFields(logrus.Fields{
+			"orphanType": orphan.Spec.Type,
+			"orphanName": orphan.Name,
+			"orphanNode": orphan.Spec.NodeID,
+		}).Infof("Orphan does not belong to this controller. Skipping resource cleanup")
+		return true, nil
 	}
 
-	if types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeDataCleanable).Status !=
-		longhorn.ConditionStatusTrue {
-		log.Infof("Only delete orphan %v resource object and do not delete the orphaned data", orphan.Name)
-		return nil
+	switch orphan.Spec.Type {
+	case longhorn.OrphanTypeEngineInstance:
+		isCleanupComplete, err = oc.cleanupOrphanedEngineInstance(orphan)
+	case longhorn.OrphanTypeReplicaInstance:
+		isCleanupComplete, err = oc.cleanupOrphanedReplicaInstance(orphan)
+	case longhorn.OrphanTypeReplicaData:
+		if types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeDataCleanable).Status !=
+			longhorn.ConditionStatusTrue {
+			log.Infof("Only delete orphan %v resource object and do not delete the orphaned data store", orphan.Name)
+			return true, nil
+		}
+		err = oc.deleteOrphanedReplicaDataStore(orphan)
+		if err == nil || (err != nil && apierrors.IsNotFound(err)) {
+			isCleanupComplete = true
+			err = nil
+		}
+	default:
+		return false, fmt.Errorf("unknown orphan type %v to clean up orphaned resource for %v", orphan.Spec.Type, orphan.Name)
 	}
 
-	switch orphan.Spec.Type {
-	case longhorn.OrphanTypeReplica:
-		err = oc.deleteOrphanedReplica(orphan)
+	return isCleanupComplete, err
+}
+
+func (oc *OrphanController) cleanupOrphanedEngineInstance(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
+	defer func() {
+		if err != nil {
+			err = errors.Wrapf(err, "cannot cleanup orphan engine instance %v", orphan.Name)
+		}
+	}()
+
+	instance, imName, err := oc.extractOrphanedInstanceInfo(orphan)
+	if err != nil {
+		return false, err
+	}
+
+	var spec *longhorn.InstanceSpec
+	if engineCR, err := oc.ds.GetEngineRO(instance); err != nil {
+		if !apierrors.IsNotFound(err) {
+			return false, err
+		}
+		spec = nil
+	} else {
+		spec = &engineCR.Spec.InstanceSpec
+	}
+	return oc.cleanupOrphanedInstance(orphan, instance, imName, longhorn.InstanceManagerTypeEngine, spec)
+}
+
+func (oc *OrphanController) cleanupOrphanedReplicaInstance(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
+	defer func() {
+		if err != nil {
+			err = errors.Wrapf(err, "cannot cleanup orphan replica instance %v", orphan.Name)
+		}
+	}()
+
+	instance, imName, err := oc.extractOrphanedInstanceInfo(orphan)
+	if err != nil {
+		return false, err
+	}
+
+	var spec *longhorn.InstanceSpec
+	if replicaCR, err := oc.ds.GetReplicaRO(instance); err != nil {
+		if !apierrors.IsNotFound(err) {
+			return false, err
+		}
+		spec = nil
+	} else {
+		spec = &replicaCR.Spec.InstanceSpec
+	}
+	return oc.cleanupOrphanedInstance(orphan, instance, imName, longhorn.InstanceManagerTypeReplica, spec)
+}
+
+func (oc *OrphanController) extractOrphanedInstanceInfo(orphan *longhorn.Orphan) (name, instanceManager string, err error) {
+	name, ok := orphan.Spec.Parameters[longhorn.OrphanInstanceName]
+	if !ok {
+		return "", "", fmt.Errorf("failed to get instance name for instance orphan %v", orphan.Name)
+	}
+
+	instanceManager, ok = orphan.Spec.Parameters[longhorn.OrphanInstanceManager]
+	if !ok {
+		return "", "", fmt.Errorf("failed to get instance manager for instance orphan %v", orphan.Name)
+	}
+
+	switch orphan.Spec.DataEngine {
+	case longhorn.DataEngineTypeV1, longhorn.DataEngineTypeV2:
+		// supported data engine type
 	default:
-		err = fmt.Errorf("unknown orphan type %v", orphan.Spec.Type)
+		return "", "", fmt.Errorf("unknown data engine type %v for instance orphan %v", orphan.Spec.DataEngine, orphan.Name)
 	}
 
-	if err == nil || (err != nil && apierrors.IsNotFound(err)) {
-		return nil
+	return name, instanceManager, nil
+}
+
+func (oc *OrphanController) cleanupOrphanedInstance(orphan *longhorn.Orphan, instance, imName string, imType longhorn.InstanceManagerType, instanceCRSpec *longhorn.InstanceSpec) (isCleanupComplete bool, err error) {
+	if instanceCRSpec != nil && instanceCRSpec.NodeID == orphan.Spec.NodeID {
+		oc.logger.Infof("Orphan instance %v is scheduled back to current node %v. Skip cleaning up the instance resource and finalize the orphan CR.", instance, orphan.Spec.NodeID)
+		return true, nil
 	}
 
-	return err
+	imc, err := oc.getRunningInstanceManagerClientForOrphan(orphan, imName)
+	if err != nil {
+		return false, errors.Wrapf(err, "cannot get running instance manager client for orphan %v", orphan.Name)
+	} else if imc == nil {
+		oc.logger.WithField("orphanInstanceNode", orphan.Spec.NodeID).Warnf("No running instance manager for deleting orphan instance %v", orphan.Name)
+		return true, nil
+	}
+	defer func() {
+		if closeErr := imc.Close(); closeErr != nil {
+			oc.logger.WithError(closeErr).Error("Failed to close instance manager client")
+		}
+	}()
+
+	if err := oc.deleteInstance(imc, instance, imType, orphan.Spec.DataEngine); err != nil {
+		return false, err
+	}
+	isCleanupComplete, cleanupErr := oc.confirmOrphanInstanceCleanup(imc, instance, imType, orphan.Spec.DataEngine)
+	if cleanupErr == nil && !isCleanupComplete {
+		oc.logger.Infof("Orphan instance %v cleanup in progress", instance)
+	}
+	return isCleanupComplete, cleanupErr
 }
 
-func (oc *OrphanController) deleteOrphanedReplica(orphan *longhorn.Orphan) error {
+func (oc *OrphanController) getRunningInstanceManagerClientForOrphan(orphan *longhorn.Orphan, imName string) (*engineapi.InstanceManagerClient, error) {
+	im, err := oc.ds.GetRunningInstanceManagerByNodeRO(orphan.Spec.NodeID, orphan.Spec.DataEngine)
+	if err != nil {
+		oc.logger.WithError(err).Warnf("No running instance manager for node %v for deleting orphan instance %v", oc.controllerID, orphan.Name)
+		return nil, nil
+	}
+	if im.Name != imName {
+		oc.logger.WithField("instanceManager", imName).Warnf("Orphan instance %v is not managed by current running instance manager %v", orphan.Name, im.Name)
+		return nil, nil
+	}
+	return engineapi.NewInstanceManagerClient(im, false)
+}
+
+func (oc *OrphanController) deleteInstance(imc *engineapi.InstanceManagerClient, instanceName string, instanceKind longhorn.InstanceManagerType, engineType longhorn.DataEngineType) (err error) {
+	// There is a delay between deletion initiation and state/InstanceManager update,
+	// this function may be called multiple times before given instance exits.
+
+	oc.logger.Infof("Orphan controller deleting instance %v", instanceName)
+
+	defer func() {
+		err = errors.Wrapf(err, "cannot delete instance %v", instanceName)
+	}()
+
+	err = imc.InstanceDelete(engineType, instanceName, string(instanceKind), "", false)
+	if err != nil && !types.ErrorIsNotFound(err) {
+		return err
+	}
+
+	return nil
+}
+
+func (oc *OrphanController) confirmOrphanInstanceCleanup(imc *engineapi.InstanceManagerClient, instanceName string, imType longhorn.InstanceManagerType, engineType longhorn.DataEngineType) (isCleanupComplete bool, err error) {
+	defer func() {
+		if err != nil {
+			err = errors.Wrapf(err, "cannot confirm cleanup result of instance %v", instanceName)
path: controller/orphan_controller.go
user: login: derekbit
body: 
```suggestion
			err = errors.Wrapf(err, "failed to confirm cleanup result of instance %v", instanceName)
```
created_at: 2025-04-23T15:00:49Z
author_association: MEMBER
diff_hunk: @@ -371,6 +575,55 @@ func (oc *OrphanController) updateConditions(orphan *longhorn.Orphan) error {
 	return nil
 }
 
+// If the InstanceState condition status is true, the corresponding instance is still exist in instance manager
+func (oc *OrphanController) updateInstanceStateCondition(orphan *longhorn.Orphan, instanceType longhorn.InstanceType) (err error) {
+	var instanceState longhorn.InstanceState
+	defer func() {
+		if err != nil {
+			return
+		}
+
+		status := longhorn.ConditionStatusTrue
+		if instanceState == longhorn.InstanceStateTerminated {
+			status = longhorn.ConditionStatusFalse
+		}
+
+		orphan.Status.Conditions = types.SetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeInstanceExist, status, string(instanceState), "")
+	}()
+
+	instanceName, instanceManager, err := oc.extractOrphanedInstanceInfo(orphan)
+	im, err := oc.ds.GetInstanceManager(instanceManager)
+	if err != nil {
+		if !apierrors.IsNotFound(err) {
path: controller/orphan_controller.go
user: login: derekbit
body: Shoud it be `if apierrors.IsNotFound(err) {`?

```suggestion
		if apierrors.IsNotFound(err) {
```
created_at: 2025-04-23T15:05:52Z
author_association: MEMBER
diff_hunk: @@ -371,6 +575,55 @@ func (oc *OrphanController) updateConditions(orphan *longhorn.Orphan) error {
 	return nil
 }
 
+// If the InstanceState condition status is true, the corresponding instance is still exist in instance manager
+func (oc *OrphanController) updateInstanceStateCondition(orphan *longhorn.Orphan, instanceType longhorn.InstanceType) (err error) {
+	var instanceState longhorn.InstanceState
+	defer func() {
+		if err != nil {
+			return
+		}
+
+		status := longhorn.ConditionStatusTrue
+		if instanceState == longhorn.InstanceStateTerminated {
+			status = longhorn.ConditionStatusFalse
+		}
+
+		orphan.Status.Conditions = types.SetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeInstanceExist, status, string(instanceState), "")
+	}()
+
+	instanceName, instanceManager, err := oc.extractOrphanedInstanceInfo(orphan)
+	im, err := oc.ds.GetInstanceManager(instanceManager)
+	if err != nil {
+		if !apierrors.IsNotFound(err) {
+			oc.logger.WithError(err).Infof("No instance manager for node %v for update instance state of orphan instance %v", oc.controllerID, orphan.Name)
+			instanceState = longhorn.InstanceStateTerminated
+			return nil
+		}
+		return err
+	}
+	imc, err := engineapi.NewInstanceManagerClient(im, false)
+	if err != nil {
+		return errors.Wrapf(err, "failed to check node down or missing manager for node %v", orphan.Spec.NodeID)
+	}
+	defer func() {
+		if closeErr := imc.Close(); closeErr != nil {
+			oc.logger.WithError(closeErr).Error("Failed to close instance manager client")
+		}
+	}()
+
+	instanceProc, err := imc.InstanceGet(orphan.Spec.DataEngine, instanceName, string(instanceType))
path: controller/orphan_controller.go
user: login: derekbit
body: 
```suggestion
	instance, err := imc.InstanceGet(orphan.Spec.DataEngine, instanceName, string(instanceType))
```
created_at: 2025-04-23T15:06:45Z
author_association: MEMBER
diff_hunk: @@ -2709,6 +2709,11 @@ spec:
             description: OrphanSpec defines the desired state of the Longhorn orphaned
               data
             properties:
+              dataEngine:
+                description: |-
+                  The type of data engine for instance orphan.
+                  Can be "v1", "v2".
+                type: string
               nodeID:
path: k8s/crds.yaml
user: login: coderabbitai[bot]
body: _🛠️ Refactor suggestion_

**CRD validation missing for `dataEngine` field**  
The new `dataEngine` property on `OrphanSpec` lacks enum and default constraints. To ensure consumers can only set `"v1"` or `"v2"` (and default to `"v1"`), add `enum` and `default` entries, consistent with other CRDs in this repo.  

```diff
              dataEngine:
                description: |-
                  The type of data engine for instance orphan.
                  Can be "v1", "v2".
-               type: string
+               type: string
+               enum:
+                 - v1
+                 - v2
+               default: v1
```


<!-- suggestion_start -->

<details>
<summary>📝 Committable suggestion</summary>

> ‼️ **IMPORTANT**
> Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

`````suggestion
              dataEngine:
                description: |-
                  The type of data engine for instance orphan.
                  Can be "v1", "v2".
                type: string
                enum:
                  - v1
                  - v2
                default: v1
              nodeID:
`````

</details>

<!-- suggestion_end -->

<!-- This is an auto-generated comment by CodeRabbit -->
created_at: 2025-04-23T15:26:26Z
author_association: NONE
diff_hunk: @@ -726,6 +746,10 @@ func GetOrphanChecksumNameForOrphanedDataStore(nodeID, diskName, diskPath, diskU
 	return orphanPrefix + util.GetStringChecksumSHA256(strings.TrimSpace(fmt.Sprintf("%s-%s-%s-%s-%s", nodeID, diskName, diskPath, diskUUID, dataStore)))
 }
 
+func GetOrphanChecksumNameForOrphanedInstance(instanceName, nodeID, dataEngine string) string {
+	return orphanPrefix + util.GetStringChecksumSHA256(strings.TrimSpace(fmt.Sprintf("%s-%s-%s", instanceName, nodeID, dataEngine)))
+}
path: types/types.go
user: login: coderabbitai[bot]
body: _⚠️ Potential issue_

**Kubernetes object‑name length could overflow**

`orphan-` + SHA256 hex = 7 + 64 = 71 chars, exceeding the 63‑character limit for Kubernetes object names.  
All other checksum helpers (`ei-…`, `imi-…`) truncate to a safe length. Do the same here to avoid creation failures on long instance/node IDs:

```diff
-return orphanPrefix + util.GetStringChecksumSHA256(...)
+const orphanInstanceChecksumLen = 48 // 7+48 = 55 < 63
+return orphanPrefix + util.GetStringChecksumSHA256(... )[:orphanInstanceChecksumLen]
```



<!-- suggestion_start -->

<details>
<summary>📝 Committable suggestion</summary>

> ‼️ **IMPORTANT**
> Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

`````suggestion
func GetOrphanChecksumNameForOrphanedInstance(instanceName, nodeID, dataEngine string) string {
    const orphanInstanceChecksumLen = 48 // 7+48 = 55 < 63
    return orphanPrefix + util.GetStringChecksumSHA256(
        strings.TrimSpace(fmt.Sprintf("%s-%s-%s", instanceName, nodeID, dataEngine)),
    )[:orphanInstanceChecksumLen]
}
`````

</details>

<!-- suggestion_end -->

<!-- This is an auto-generated reply by CodeRabbit -->
created_at: 2025-04-23T15:26:27Z
author_association: NONE
diff_hunk: @@ -85,3 +101,53 @@ func upgradeVolumes(namespace string, lhClient *lhclientset.Clientset, resourceM
 
 	return nil
 }
+
+func upgradeSettings(namespace string, lhClient *lhclientset.Clientset, resourceMaps map[string]interface{}) (err error) {
+	defer func() {
+		err = errors.Wrapf(err, upgradeLogPrefix+"upgrade settings failed")
+	}()
+
+	settingsMap, err := upgradeutil.ListAndUpdateSettingsInProvidedCache(namespace, lhClient, resourceMaps)
+	if err != nil {
+		if apierrors.IsNotFound(err) {
+			return nil
+		}
+		return errors.Wrapf(err, "failed to list all existing Longhorn settings during the settings upgrade")
+	}
+
+	if err := updateOrphanResourceAutoDeletionSetting(settingsMap); err != nil {
+		return errors.Wrapf(err, "failed to update orphan resource auto-deletion settings")
+	}
+
+	return nil
+}
+
+func updateOrphanResourceAutoDeletionSetting(settingsMap map[string]*v1beta2.Setting) (err error) {
+	oldSetting, exist := settingsMap[string(types.SettingNameOrphanAutoDeletion)]
+	if !exist {
+		return nil
+	}
+	newSetting, exist := settingsMap[string(types.SettingNameOrphanResourceAutoDeletion)]
+	if !exist {
+		newSetting = &v1beta2.Setting{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: string(types.SettingNameOrphanResourceAutoDeletion),
+			},
+		}
+		settingsMap[string(types.SettingNameOrphanResourceAutoDeletion)] = newSetting
+	}
+	resourceTypes, err := types.UnmarshalOrphanResourceTypes(newSetting.Value)
+	if err != nil {
+		return err
+	}
+
+	resourceTypes[types.OrphanResourceTypeReplicaData] = oldSetting.Value == "true"
+	enabledResourceType := make([]string, 0, len(resourceTypes))
+	for rt, enabled := range resourceTypes {
+		if enabled {
+			enabledResourceType = append(enabledResourceType, string(rt))
+		}
+	}
+	newSetting.Value = strings.Join(enabledResourceType, ";")
path: upgrade/v18xto190/upgrade.go
user: login: coderabbitai[bot]
body: _⚠️ Potential issue_

**Possible nil‑map panic when new setting is absent**

`UnmarshalOrphanResourceTypes` might return `nil` when `newSetting.Value` is empty (first‑install scenario).  
The subsequent write `resourceTypes[types.OrphanResourceTypeReplicaData] = …` will panic.

```diff
resourceTypes, err := types.UnmarshalOrphanResourceTypes(newSetting.Value)
if err != nil {
    return err
}
+if resourceTypes == nil {
+    resourceTypes = map[types.OrphanResourceType]bool{}
+}
```




<!-- suggestion_start -->

<details>
<summary>📝 Committable suggestion</summary>

> ‼️ **IMPORTANT**
> Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

`````suggestion
resourceTypes, err := types.UnmarshalOrphanResourceTypes(newSetting.Value)
if err != nil {
    return err
}
if resourceTypes == nil {
    resourceTypes = map[types.OrphanResourceType]bool{}
}

resourceTypes[types.OrphanResourceTypeReplicaData] = oldSetting.Value == "true"
enabledResourceType := make([]string, 0, len(resourceTypes))
for rt, enabled := range resourceTypes {
    if enabled {
        enabledResourceType = append(enabledResourceType, string(rt))
    }
}
newSetting.Value = strings.Join(enabledResourceType, ";")
`````

</details>

<!-- suggestion_end -->

<!-- This is an auto-generated reply by CodeRabbit -->
created_at: 2025-04-23T15:26:27Z
author_association: NONE
diff_hunk: @@ -269,48 +303,205 @@
 	return isControllerResponsibleFor(oc.controllerID, oc.ds, orphan.Name, orphan.Spec.NodeID, orphan.Status.OwnerID)
 }
 
-func (oc *OrphanController) cleanupOrphanedData(orphan *longhorn.Orphan) (err error) {
+func (oc *OrphanController) cleanupOrphanedResource(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
 	log := getLoggerForOrphan(oc.logger, orphan)
 
 	defer func() {
 		if err == nil {
 			return
 		}
 
-		err = errors.Wrapf(err, "failed to delete orphan %v data", orphan.Name)
+		err = errors.Wrapf(err, "failed to delete %v orphan %v", orphan.Spec.Type, orphan.Name)
 		orphan.Status.Conditions = types.SetCondition(orphan.Status.Conditions,
 			longhorn.OrphanConditionTypeError, longhorn.ConditionStatusTrue, "", err.Error())
 	}()
 
 	// Make sure if the orphan nodeID and controller ID are same.
 	// If NO, just delete the orphan resource object and don't touch the data.
 	if orphan.Spec.NodeID != oc.controllerID {
-		log.Infof("Orphan nodeID %v is different from controllerID %v, so just delete the orphan resource object",
-			orphan.Name, oc.controllerID)
-		return nil
+		log.WithFields(logrus.Fields{
+			"orphanType": orphan.Spec.Type,
+			"orphanName": orphan.Name,
+			"orphanNode": orphan.Spec.NodeID,
+		}).Infof("Orphan does not belong to this controller. Skipping resource cleanup")
+		return true, nil
 	}
 
-	if types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeDataCleanable).Status !=
-		longhorn.ConditionStatusTrue {
-		log.Infof("Only delete orphan %v resource object and do not delete the orphaned data", orphan.Name)
-		return nil
+	switch orphan.Spec.Type {
+	case longhorn.OrphanTypeEngineInstance:
+		isCleanupComplete, err = oc.cleanupOrphanedEngineInstance(orphan)
+	case longhorn.OrphanTypeReplicaInstance:
+		isCleanupComplete, err = oc.cleanupOrphanedReplicaInstance(orphan)
+	case longhorn.OrphanTypeReplicaData:
+		if types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeDataCleanable).Status !=
+			longhorn.ConditionStatusTrue {
+			log.Infof("Only delete orphan %v resource object and do not delete the orphaned data store", orphan.Name)
+			return true, nil
+		}
+		err = oc.deleteOrphanedReplicaDataStore(orphan)
+		if err == nil || (err != nil && apierrors.IsNotFound(err)) {
+			isCleanupComplete = true
+			err = nil
+		}
+	default:
+		return false, fmt.Errorf("unknown orphan type %v to clean up orphaned resource for %v", orphan.Spec.Type, orphan.Name)
 	}
 
-	switch orphan.Spec.Type {
-	case longhorn.OrphanTypeReplica:
-		err = oc.deleteOrphanedReplica(orphan)
+	return isCleanupComplete, err
+}
+
+func (oc *OrphanController) cleanupOrphanedEngineInstance(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
+	defer func() {
+		if err != nil {
+			err = errors.Wrapf(err, "cannot cleanup orphan engine instance %v", orphan.Name)
+		}
+	}()
+
+	instance, imName, err := oc.extractOrphanedInstanceInfo(orphan)
+	if err != nil {
+		return false, err
+	}
+
+	var spec *longhorn.InstanceSpec
+	if engineCR, err := oc.ds.GetEngineRO(instance); err != nil {
+		if !apierrors.IsNotFound(err) {
+			return false, err
+		}
+		spec = nil
+	} else {
+		spec = &engineCR.Spec.InstanceSpec
+	}
+	return oc.cleanupOrphanedInstance(orphan, instance, imName, longhorn.InstanceManagerTypeEngine, spec)
+}
+
+func (oc *OrphanController) cleanupOrphanedReplicaInstance(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
+	defer func() {
+		if err != nil {
+			err = errors.Wrapf(err, "cannot cleanup orphan replica instance %v", orphan.Name)
+		}
+	}()
+
+	instance, imName, err := oc.extractOrphanedInstanceInfo(orphan)
+	if err != nil {
+		return false, err
+	}
+
+	var spec *longhorn.InstanceSpec
+	if replicaCR, err := oc.ds.GetReplicaRO(instance); err != nil {
+		if !apierrors.IsNotFound(err) {
+			return false, err
+		}
+		spec = nil
+	} else {
+		spec = &replicaCR.Spec.InstanceSpec
+	}
+	return oc.cleanupOrphanedInstance(orphan, instance, imName, longhorn.InstanceManagerTypeReplica, spec)
+}
+
+func (oc *OrphanController) extractOrphanedInstanceInfo(orphan *longhorn.Orphan) (name, instanceManager string, err error) {
+	name, ok := orphan.Spec.Parameters[longhorn.OrphanInstanceName]
+	if !ok {
+		return "", "", fmt.Errorf("failed to get instance name for instance orphan %v", orphan.Name)
+	}
+
+	instanceManager, ok = orphan.Spec.Parameters[longhorn.OrphanInstanceManager]
+	if !ok {
+		return "", "", fmt.Errorf("failed to get instance manager for instance orphan %v", orphan.Name)
+	}
+
+	switch orphan.Spec.DataEngine {
+	case longhorn.DataEngineTypeV1, longhorn.DataEngineTypeV2:
+		// supported data engine type
 	default:
-		err = fmt.Errorf("unknown orphan type %v", orphan.Spec.Type)
+		return "", "", fmt.Errorf("unknown data engine type %v for instance orphan %v", orphan.Spec.DataEngine, orphan.Name)
 	}
 
-	if err == nil || (err != nil && apierrors.IsNotFound(err)) {
-		return nil
+	return name, instanceManager, nil
+}
+
+func (oc *OrphanController) cleanupOrphanedInstance(orphan *longhorn.Orphan, instance, imName string, imType longhorn.InstanceManagerType, instanceCRSpec *longhorn.InstanceSpec) (isCleanupComplete bool, err error) {
+	if instanceCRSpec != nil && instanceCRSpec.NodeID == orphan.Spec.NodeID {
+		oc.logger.Infof("Orphan instance %v is scheduled back to current node %v. Skip cleaning up the instance resource and finalize the orphan CR.", instance, orphan.Spec.NodeID)
+		return true, nil
 	}
 
-	return err
+	imc, err := oc.getRunningInstanceManagerClientForOrphan(orphan, imName)
+	if err != nil {
+		return false, errors.Wrapf(err, "cannot get running instance manager client for orphan %v", orphan.Name)
+	} else if imc == nil {
+		oc.logger.WithField("orphanInstanceNode", orphan.Spec.NodeID).Warnf("No running instance manager for deleting orphan instance %v", orphan.Name)
+		return true, nil
+	}
+	defer func() {
+		if closeErr := imc.Close(); closeErr != nil {
+			oc.logger.WithError(closeErr).Error("Failed to close instance manager client")
+		}
+	}()
+
+	if err := oc.deleteInstance(imc, instance, imType, orphan.Spec.DataEngine); err != nil {
+		return false, err
+	}
+	isCleanupComplete, cleanupErr := oc.confirmOrphanInstanceCleanup(imc, instance, imType, orphan.Spec.DataEngine)
+	if cleanupErr == nil && !isCleanupComplete {
+		oc.logger.Infof("Orphan instance %v cleanup in progress", instance)
+	}
+	return isCleanupComplete, cleanupErr
 }
 
-func (oc *OrphanController) deleteOrphanedReplica(orphan *longhorn.Orphan) error {
+func (oc *OrphanController) getRunningInstanceManagerClientForOrphan(orphan *longhorn.Orphan, imName string) (*engineapi.InstanceManagerClient, error) {
+	im, err := oc.ds.GetRunningInstanceManagerByNodeRO(orphan.Spec.NodeID, orphan.Spec.DataEngine)
+	if err != nil {
+		oc.logger.WithError(err).Warnf("No running instance manager for node %v for deleting orphan instance %v", oc.controllerID, orphan.Name)
+		return nil, nil
+	}
+	if im.Name != imName {
+		oc.logger.WithField("instanceManager", imName).Warnf("Orphan instance %v is not managed by current running instance manager %v", orphan.Name, im.Name)
+		return nil, nil
+	}
path: controller/orphan_controller.go
user: login: coderabbitai[bot]
body: _🛠️ Refactor suggestion_

**Swallowing datastore errors may hide real problems**

`getRunningInstanceManagerClientForOrphan` treats any error from
`ds.GetRunningInstanceManagerByNodeRO(...)` as “manager not found”.  
If the datastore returned a transient etcd error we’d silently ignore it and mark
the orphan cleaned‑up, which can lead to leaked SPDK processes.

Consider distinguishing:

* `apierrors.IsNotFound` → skip deletion  
* everything else           → requeue with error

<!-- This is an auto-generated comment by CodeRabbit -->
created_at: 2025-04-23T15:26:27Z
author_association: NONE
diff_hunk: @@ -371,6 +575,55 @@
 	return nil
 }
 
+// If the InstanceState condition status is true, the corresponding instance is still exist in instance manager
+func (oc *OrphanController) updateInstanceStateCondition(orphan *longhorn.Orphan, instanceType longhorn.InstanceType) (err error) {
+	var instanceState longhorn.InstanceState
+	defer func() {
+		if err != nil {
+			return
+		}
+
+		status := longhorn.ConditionStatusTrue
+		if instanceState == longhorn.InstanceStateTerminated {
+			status = longhorn.ConditionStatusFalse
+		}
+
+		orphan.Status.Conditions = types.SetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeInstanceExist, status, string(instanceState), "")
+	}()
+
+	instanceName, instanceManager, err := oc.extractOrphanedInstanceInfo(orphan)
+	im, err := oc.ds.GetInstanceManager(instanceManager)
+	if err != nil {
+		if !apierrors.IsNotFound(err) {
+			oc.logger.WithError(err).Infof("No instance manager for node %v for update instance state of orphan instance %v", oc.controllerID, orphan.Name)
+			instanceState = longhorn.InstanceStateTerminated
+			return nil
+		}
+		return err
+	}
path: controller/orphan_controller.go
user: login: coderabbitai[bot]
body: _⚠️ Potential issue_

**Error from `extractOrphanedInstanceInfo()` is silently discarded due to shadowing**

`:=` creates a new `err` that shadows the named return variable, so if extraction fails we still continue with an empty `instanceManager`, hitting further errors or nil dereferences.

```diff
-instanceName, instanceManager, err := oc.extractOrphanedInstanceInfo(orphan)
-if err != nil {
-    // err is the shadowed variable – lost on return
-}
+instanceName, instanceManager, extractErr := oc.extractOrphanedInstanceInfo(orphan)
+if extractErr != nil {
+    return extractErr
+}
```

The same shadowing pattern appears in *both* instance‑cleanup helpers; consider a project‑wide sweep.  



> Committable suggestion skipped: line range outside the PR's diff.

<!-- This is an auto-generated comment by CodeRabbit -->
created_at: 2025-04-23T15:26:27Z
author_association: NONE
diff_hunk: @@ -170,6 +172,13 @@
 	}
 	imc.cacheSyncs = append(imc.cacheSyncs, ds.KubeNodeInformer.HasSynced)
 
+	if _, err = ds.OrphanInformer.AddEventHandlerWithResyncPeriod(cache.ResourceEventHandlerFuncs{
+		AddFunc: imc.enqueueInstanceManagerOrphan,
+	}, 0); err != nil {
+		return nil, err
+	}
+	imc.cacheSyncs = append(imc.cacheSyncs, ds.OrphanInformer.HasSynced)
+
path: controller/instance_manager_controller.go
user: login: coderabbitai[bot]
body: _🛠️ Refactor suggestion_

**Informer only reacts to Orphan *additions***  

You enqueue instance‑manager reconciliation on `AddFunc` but ignore
orphan updates and deletions.  Deleting an orphan should also trigger a
re‑sync so the controller can clean up its cache and potentially create
replacement orphans.  Consider registering `UpdateFunc` and `DeleteFunc`
as well.

```go
ds.OrphanInformer.AddEventHandlerWithResyncPeriod(cache.ResourceEventHandlerFuncs{
    AddFunc:    imc.enqueueInstanceManagerOrphan,
    UpdateFunc: func(_, obj interface{}) { imc.enqueueInstanceManagerOrphan(obj) },
    DeleteFunc: imc.enqueueInstanceManagerOrphan,
}, 0)
```

<!-- This is an auto-generated reply by CodeRabbit -->
created_at: 2025-04-23T15:26:27Z
author_association: NONE
diff_hunk: @@ -1610,6 +1650,123 @@
 	return podSpec, nil
 }
 
+func (imc *InstanceManagerController) deleteOrphans(im *longhorn.InstanceManager, isInstanceManagerTerminating bool) error {
+	autoDeletionTypes, err := imc.ds.GetSettingOrphanResourceAutoDeletion()
+	var autoDeleteEngine = false
+	var autoDeleteReplica = false
+	if err != nil {
+		imc.logger.WithError(err).Warnf("Failed to fetch orphan auto deletion setting, disabled by default")
+	} else {
+		autoDeleteEngine = autoDeletionTypes[types.OrphanResourceTypeEngineInstance]
+		autoDeleteReplica = autoDeletionTypes[types.OrphanResourceTypeReplicaInstance]
+	}
+
+	orphanList, err := imc.ds.ListOrphansForEngineAndReplicaInstancesRO(im.Spec.NodeID)
+	if err != nil {
+		return err
+	}
+	multiError := util.NewMultiError()
+	for _, orphan := range orphanList {
+		if !orphan.DeletionTimestamp.IsZero() {
+			continue
+		}
path: controller/instance_manager_controller.go
user: login: coderabbitai[bot]
body: _⚠️ Potential issue_

**Potential nil‑pointer panic when checking `DeletionTimestamp`**  

`DeletionTimestamp` is a pointer; dereferencing it without a nil‑check can panic the controller when an orphan has never scheduled for deletion.

```diff
-        if !orphan.DeletionTimestamp.IsZero() {
+        if orphan.DeletionTimestamp != nil && !orphan.DeletionTimestamp.IsZero() {
             continue
         }
```



<!-- suggestion_start -->

<details>
<summary>📝 Committable suggestion</summary>

> ‼️ **IMPORTANT**
> Carefully review the code before committing. Ensure that it accurately replaces the highlighted code, contains no missing lines, and has no issues with indentation. Thoroughly test & benchmark the code to ensure it meets the requirements.

`````suggestion
		if orphan.DeletionTimestamp != nil && !orphan.DeletionTimestamp.IsZero() {
			continue
		}
`````

</details>

<!-- suggestion_end -->

<!-- This is an auto-generated reply by CodeRabbit -->
created_at: 2025-04-23T15:26:27Z
author_association: NONE
diff_hunk: @@ -1610,6 +1650,123 @@
 	return podSpec, nil
 }
 
+func (imc *InstanceManagerController) deleteOrphans(im *longhorn.InstanceManager, isInstanceManagerTerminating bool) error {
+	autoDeletionTypes, err := imc.ds.GetSettingOrphanResourceAutoDeletion()
+	var autoDeleteEngine = false
+	var autoDeleteReplica = false
+	if err != nil {
+		imc.logger.WithError(err).Warnf("Failed to fetch orphan auto deletion setting, disabled by default")
+	} else {
+		autoDeleteEngine = autoDeletionTypes[types.OrphanResourceTypeEngineInstance]
+		autoDeleteReplica = autoDeletionTypes[types.OrphanResourceTypeReplicaInstance]
+	}
+
+	orphanList, err := imc.ds.ListOrphansForEngineAndReplicaInstancesRO(im.Spec.NodeID)
+	if err != nil {
+		return err
+	}
+	multiError := util.NewMultiError()
+	for _, orphan := range orphanList {
+		if !orphan.DeletionTimestamp.IsZero() {
+			continue
+		}
+
+		var isDeletable = false
+		switch orphan.Spec.Type {
+		case longhorn.OrphanTypeEngineInstance:
+			isDeletable, err = imc.isOrphanDeletable(orphan, isInstanceManagerTerminating || autoDeleteEngine, im.Status.InstanceEngines, imc.isEngineScheduledOnCurrentNode)
+		case longhorn.OrphanTypeReplicaInstance:
+			isDeletable, err = imc.isOrphanDeletable(orphan, isInstanceManagerTerminating || autoDeleteReplica, im.Status.InstanceReplicas, imc.isReplicaScheduledOnCurrentNode)
+		}
+		if err != nil {
+			multiError.Append(util.NewMultiError(fmt.Sprintf("%v: %v", orphan.Name, err)))
+		} else if isDeletable {
+			if err := imc.ds.DeleteOrphan(orphan.Name); err != nil && !apierrors.IsNotFound(err) {
+				multiError.Append(util.NewMultiError(fmt.Sprintf("%v: %v", orphan.Name, err)))
+			}
+		}
+	}
+	if len(multiError) > 0 {
+		return fmt.Errorf("instance manager failed to delete orphan CR: %v", multiError.Join())
+	}
+	return nil
+}
+
+func (imc *InstanceManagerController) isOrphanDeletable(orphan *longhorn.Orphan, autoDelete bool, instanceMap instanceProcessMap, isInstanceScheduledOnNode func(node, instance string) (bool, error)) (isDeletable bool, err error) {
+	if autoDelete {
+		return true, nil
+	}
+
+	instanceName := orphan.Spec.Parameters[longhorn.OrphanInstanceName]
+	instanceProc, instanceExist := instanceMap[instanceName]
+	instanceCRScheduledBack, err := isInstanceScheduledOnNode(orphan.Spec.NodeID, instanceName)
+	if err != nil {
+		return false, errors.Wrapf(err, "failed to check CR scheduled node for %v orphan %v", orphan.Spec.Type, instanceName)
+	}
+
+	var instanceState longhorn.InstanceState
+	if instanceExist {
+		instanceState = instanceProc.Status.State
+	} else {
+		instanceState = longhorn.InstanceStateTerminated
+	}
+
+	imc.logger.WithFields(logrus.Fields{
+		"instanceExist":           instanceExist,
+		"instanceState":           instanceState,
+		"instanceCRScheduledBack": instanceCRScheduledBack,
+	}).Debugf("Checked deletable for %s orphan instance %s", orphan.Spec.Type, orphan.Name)
+	if !instanceExist || instanceCRScheduledBack {
+		return true, nil
+	}
+	return false, nil
+}
+
+func (imc *InstanceManagerController) isEngineScheduledOnCurrentNode(node string, instance string) (bool, error) {
+	existEngine, err := imc.ds.GetEngineRO(instance)
+	switch {
+	case err == nil:
+		// Engine CR still exists - check the ownership
+		return imc.isInstanceScheduledOnCurrentNode(node, &existEngine.Spec.InstanceSpec, &existEngine.Status.InstanceStatus), nil
+	case apierrors.IsNotFound(err):
+		// Engine CR not found - instance is orphaned
+		return false, nil
+	default:
+		// Unexpected error - unable to check if engine instance is orphaned or not
+		return false, err
+	}
+}
+
+func (imc *InstanceManagerController) isReplicaScheduledOnCurrentNode(node string, instance string) (bool, error) {
+	existReplica, err := imc.ds.GetReplicaRO(instance)
+	switch {
+	case err == nil:
+		// Replica CR still exists - check the ownership
+		return imc.isInstanceScheduledOnCurrentNode(node, &existReplica.Spec.InstanceSpec, &existReplica.Status.InstanceStatus), nil
+	case apierrors.IsNotFound(err):
+		// Replica CR not found - instance is orphaned
+		return false, nil
+	default:
+		// Unexpected error - unable to check if replica instance is orphaned or not
+		return false, err
+	}
+}
+
+// isInstanceScheduledOnCurrentNode returns true only when it is very certain that an instance is scheduled on this node
+func (imc *InstanceManagerController) isInstanceScheduledOnCurrentNode(node string, spec *longhorn.InstanceSpec, status *longhorn.InstanceStatus) bool {
+	if status.CurrentState != spec.DesireState {
+		return false
+	}
+	switch status.CurrentState {
+	case longhorn.InstanceStateRunning:
+		return status.OwnerID == spec.NodeID && spec.NodeID == node
+	case longhorn.InstanceStateStopped:
+		return spec.NodeID == imc.controllerID
+	default:
path: controller/instance_manager_controller.go
user: login: coderabbitai[bot]
body: _🛠️ Refactor suggestion_

**Stopped‑state ownership test ignores the supplied `node` argument**  

For `InstanceStateStopped` you compare `spec.NodeID` with `imc.controllerID`, *not* the `node` function argument.  
This causes `isInstanceScheduledOnCurrentNode()` to return `true` even when the stopped instance belongs to a
different node but the same controller happens to run on multiple nodes (e.g. during upgrade/lead‑switch).

Consider:

```diff
-    case longhorn.InstanceStateStopped:
-        return spec.NodeID == imc.controllerID
+    case longhorn.InstanceStateStopped:
+        return spec.NodeID == node
```

This keeps the semantic “scheduled **on that node**” independent of which manager currently owns the controller ID.

<!-- This is an auto-generated comment by CodeRabbit -->
created_at: 2025-04-23T15:26:27Z
author_association: NONE
diff_hunk: @@ -371,6 +575,55 @@ func (oc *OrphanController) updateConditions(orphan *longhorn.Orphan) error {
 	return nil
 }
 
+// If the InstanceState condition status is true, the corresponding instance is still exist in instance manager
+func (oc *OrphanController) updateInstanceStateCondition(orphan *longhorn.Orphan, instanceType longhorn.InstanceType) (err error) {
+	var instanceState longhorn.InstanceState
+	defer func() {
+		if err != nil {
+			return
+		}
+
+		status := longhorn.ConditionStatusTrue
+		if instanceState == longhorn.InstanceStateTerminated {
+			status = longhorn.ConditionStatusFalse
+		}
+
+		orphan.Status.Conditions = types.SetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeInstanceExist, status, string(instanceState), "")
+	}()
+
+	instanceName, instanceManager, err := oc.extractOrphanedInstanceInfo(orphan)
+	im, err := oc.ds.GetInstanceManager(instanceManager)
+	if err != nil {
+		if !apierrors.IsNotFound(err) {
path: controller/orphan_controller.go
user: login: COLDTURNIP
body: Sure, it should be `apierrors.IsNotFound(err)`
created_at: 2025-04-24T01:16:18Z
author_association: MEMBER
diff_hunk: @@ -269,48 +303,205 @@ func (oc *OrphanController) isResponsibleFor(orphan *longhorn.Orphan) bool {
 	return isControllerResponsibleFor(oc.controllerID, oc.ds, orphan.Name, orphan.Spec.NodeID, orphan.Status.OwnerID)
 }
 
-func (oc *OrphanController) cleanupOrphanedData(orphan *longhorn.Orphan) (err error) {
+func (oc *OrphanController) cleanupOrphanedResource(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
 	log := getLoggerForOrphan(oc.logger, orphan)
 
 	defer func() {
 		if err == nil {
 			return
 		}
 
-		err = errors.Wrapf(err, "failed to delete orphan %v data", orphan.Name)
+		err = errors.Wrapf(err, "failed to delete %v orphan %v", orphan.Spec.Type, orphan.Name)
 		orphan.Status.Conditions = types.SetCondition(orphan.Status.Conditions,
 			longhorn.OrphanConditionTypeError, longhorn.ConditionStatusTrue, "", err.Error())
 	}()
 
 	// Make sure if the orphan nodeID and controller ID are same.
 	// If NO, just delete the orphan resource object and don't touch the data.
 	if orphan.Spec.NodeID != oc.controllerID {
-		log.Infof("Orphan nodeID %v is different from controllerID %v, so just delete the orphan resource object",
-			orphan.Name, oc.controllerID)
-		return nil
+		log.WithFields(logrus.Fields{
+			"orphanType": orphan.Spec.Type,
+			"orphanName": orphan.Name,
+			"orphanNode": orphan.Spec.NodeID,
+		}).Infof("Orphan does not belong to this controller. Skipping resource cleanup")
+		return true, nil
 	}
 
-	if types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeDataCleanable).Status !=
-		longhorn.ConditionStatusTrue {
-		log.Infof("Only delete orphan %v resource object and do not delete the orphaned data", orphan.Name)
-		return nil
+	switch orphan.Spec.Type {
+	case longhorn.OrphanTypeEngineInstance:
+		isCleanupComplete, err = oc.cleanupOrphanedEngineInstance(orphan)
+	case longhorn.OrphanTypeReplicaInstance:
+		isCleanupComplete, err = oc.cleanupOrphanedReplicaInstance(orphan)
+	case longhorn.OrphanTypeReplicaData:
+		if types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeDataCleanable).Status !=
+			longhorn.ConditionStatusTrue {
+			log.Infof("Only delete orphan %v resource object and do not delete the orphaned data store", orphan.Name)
+			return true, nil
+		}
+		err = oc.deleteOrphanedReplicaDataStore(orphan)
+		if err == nil || (err != nil && apierrors.IsNotFound(err)) {
+			isCleanupComplete = true
+			err = nil
+		}
+	default:
+		return false, fmt.Errorf("unknown orphan type %v to clean up orphaned resource for %v", orphan.Spec.Type, orphan.Name)
 	}
 
-	switch orphan.Spec.Type {
-	case longhorn.OrphanTypeReplica:
-		err = oc.deleteOrphanedReplica(orphan)
+	return isCleanupComplete, err
+}
+
+func (oc *OrphanController) cleanupOrphanedEngineInstance(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
+	defer func() {
+		if err != nil {
+			err = errors.Wrapf(err, "cannot cleanup orphan engine instance %v", orphan.Name)
path: controller/orphan_controller.go
user: login: derekbit
body: ```suggestion
			err = errors.Wrapf(err, "failed to cleanup orphan engine instance %v", orphan.Name)
```
created_at: 2025-04-24T01:20:07Z
author_association: MEMBER
diff_hunk: @@ -269,48 +303,205 @@ func (oc *OrphanController) isResponsibleFor(orphan *longhorn.Orphan) bool {
 	return isControllerResponsibleFor(oc.controllerID, oc.ds, orphan.Name, orphan.Spec.NodeID, orphan.Status.OwnerID)
 }
 
-func (oc *OrphanController) cleanupOrphanedData(orphan *longhorn.Orphan) (err error) {
+func (oc *OrphanController) cleanupOrphanedResource(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
 	log := getLoggerForOrphan(oc.logger, orphan)
 
 	defer func() {
 		if err == nil {
 			return
 		}
 
-		err = errors.Wrapf(err, "failed to delete orphan %v data", orphan.Name)
+		err = errors.Wrapf(err, "failed to delete %v orphan %v", orphan.Spec.Type, orphan.Name)
 		orphan.Status.Conditions = types.SetCondition(orphan.Status.Conditions,
 			longhorn.OrphanConditionTypeError, longhorn.ConditionStatusTrue, "", err.Error())
 	}()
 
 	// Make sure if the orphan nodeID and controller ID are same.
 	// If NO, just delete the orphan resource object and don't touch the data.
 	if orphan.Spec.NodeID != oc.controllerID {
-		log.Infof("Orphan nodeID %v is different from controllerID %v, so just delete the orphan resource object",
-			orphan.Name, oc.controllerID)
-		return nil
+		log.WithFields(logrus.Fields{
+			"orphanType": orphan.Spec.Type,
+			"orphanName": orphan.Name,
+			"orphanNode": orphan.Spec.NodeID,
+		}).Infof("Orphan does not belong to this controller. Skipping resource cleanup")
+		return true, nil
 	}
 
-	if types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeDataCleanable).Status !=
-		longhorn.ConditionStatusTrue {
-		log.Infof("Only delete orphan %v resource object and do not delete the orphaned data", orphan.Name)
-		return nil
+	switch orphan.Spec.Type {
+	case longhorn.OrphanTypeEngineInstance:
+		isCleanupComplete, err = oc.cleanupOrphanedEngineInstance(orphan)
+	case longhorn.OrphanTypeReplicaInstance:
+		isCleanupComplete, err = oc.cleanupOrphanedReplicaInstance(orphan)
+	case longhorn.OrphanTypeReplicaData:
+		if types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeDataCleanable).Status !=
+			longhorn.ConditionStatusTrue {
+			log.Infof("Only delete orphan %v resource object and do not delete the orphaned data store", orphan.Name)
+			return true, nil
+		}
+		err = oc.deleteOrphanedReplicaDataStore(orphan)
+		if err == nil || (err != nil && apierrors.IsNotFound(err)) {
+			isCleanupComplete = true
+			err = nil
+		}
+	default:
+		return false, fmt.Errorf("unknown orphan type %v to clean up orphaned resource for %v", orphan.Spec.Type, orphan.Name)
 	}
 
-	switch orphan.Spec.Type {
-	case longhorn.OrphanTypeReplica:
-		err = oc.deleteOrphanedReplica(orphan)
+	return isCleanupComplete, err
+}
+
+func (oc *OrphanController) cleanupOrphanedEngineInstance(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
+	defer func() {
+		if err != nil {
+			err = errors.Wrapf(err, "cannot cleanup orphan engine instance %v", orphan.Name)
+		}
+	}()
+
+	instance, imName, err := oc.extractOrphanedInstanceInfo(orphan)
+	if err != nil {
+		return false, err
+	}
+
+	var spec *longhorn.InstanceSpec
+	if engineCR, err := oc.ds.GetEngineRO(instance); err != nil {
+		if !apierrors.IsNotFound(err) {
+			return false, err
+		}
+		spec = nil
+	} else {
+		spec = &engineCR.Spec.InstanceSpec
+	}
+	return oc.cleanupOrphanedInstance(orphan, instance, imName, longhorn.InstanceManagerTypeEngine, spec)
+}
+
+func (oc *OrphanController) cleanupOrphanedReplicaInstance(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
+	defer func() {
+		if err != nil {
+			err = errors.Wrapf(err, "cannot cleanup orphan replica instance %v", orphan.Name)
path: controller/orphan_controller.go
user: login: derekbit
body: ```suggestion
			err = errors.Wrapf(err, "failed to cleanup orphan replica instance %v", orphan.Name)
```
created_at: 2025-04-24T01:20:20Z
author_association: MEMBER
diff_hunk: @@ -269,48 +303,205 @@ func (oc *OrphanController) isResponsibleFor(orphan *longhorn.Orphan) bool {
 	return isControllerResponsibleFor(oc.controllerID, oc.ds, orphan.Name, orphan.Spec.NodeID, orphan.Status.OwnerID)
 }
 
-func (oc *OrphanController) cleanupOrphanedData(orphan *longhorn.Orphan) (err error) {
+func (oc *OrphanController) cleanupOrphanedResource(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
 	log := getLoggerForOrphan(oc.logger, orphan)
 
 	defer func() {
 		if err == nil {
 			return
 		}
 
-		err = errors.Wrapf(err, "failed to delete orphan %v data", orphan.Name)
+		err = errors.Wrapf(err, "failed to delete %v orphan %v", orphan.Spec.Type, orphan.Name)
 		orphan.Status.Conditions = types.SetCondition(orphan.Status.Conditions,
 			longhorn.OrphanConditionTypeError, longhorn.ConditionStatusTrue, "", err.Error())
 	}()
 
 	// Make sure if the orphan nodeID and controller ID are same.
 	// If NO, just delete the orphan resource object and don't touch the data.
 	if orphan.Spec.NodeID != oc.controllerID {
-		log.Infof("Orphan nodeID %v is different from controllerID %v, so just delete the orphan resource object",
-			orphan.Name, oc.controllerID)
-		return nil
+		log.WithFields(logrus.Fields{
+			"orphanType": orphan.Spec.Type,
+			"orphanName": orphan.Name,
+			"orphanNode": orphan.Spec.NodeID,
+		}).Infof("Orphan does not belong to this controller. Skipping resource cleanup")
+		return true, nil
 	}
 
-	if types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeDataCleanable).Status !=
-		longhorn.ConditionStatusTrue {
-		log.Infof("Only delete orphan %v resource object and do not delete the orphaned data", orphan.Name)
-		return nil
+	switch orphan.Spec.Type {
+	case longhorn.OrphanTypeEngineInstance:
+		isCleanupComplete, err = oc.cleanupOrphanedEngineInstance(orphan)
+	case longhorn.OrphanTypeReplicaInstance:
+		isCleanupComplete, err = oc.cleanupOrphanedReplicaInstance(orphan)
+	case longhorn.OrphanTypeReplicaData:
+		if types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeDataCleanable).Status !=
+			longhorn.ConditionStatusTrue {
+			log.Infof("Only delete orphan %v resource object and do not delete the orphaned data store", orphan.Name)
+			return true, nil
+		}
+		err = oc.deleteOrphanedReplicaDataStore(orphan)
+		if err == nil || (err != nil && apierrors.IsNotFound(err)) {
+			isCleanupComplete = true
+			err = nil
+		}
+	default:
+		return false, fmt.Errorf("unknown orphan type %v to clean up orphaned resource for %v", orphan.Spec.Type, orphan.Name)
 	}
 
-	switch orphan.Spec.Type {
-	case longhorn.OrphanTypeReplica:
-		err = oc.deleteOrphanedReplica(orphan)
+	return isCleanupComplete, err
+}
+
+func (oc *OrphanController) cleanupOrphanedEngineInstance(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
+	defer func() {
+		if err != nil {
+			err = errors.Wrapf(err, "cannot cleanup orphan engine instance %v", orphan.Name)
+		}
+	}()
+
+	instance, imName, err := oc.extractOrphanedInstanceInfo(orphan)
+	if err != nil {
+		return false, err
+	}
+
+	var spec *longhorn.InstanceSpec
+	if engineCR, err := oc.ds.GetEngineRO(instance); err != nil {
+		if !apierrors.IsNotFound(err) {
+			return false, err
+		}
+		spec = nil
+	} else {
+		spec = &engineCR.Spec.InstanceSpec
+	}
+	return oc.cleanupOrphanedInstance(orphan, instance, imName, longhorn.InstanceManagerTypeEngine, spec)
+}
+
+func (oc *OrphanController) cleanupOrphanedReplicaInstance(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
+	defer func() {
+		if err != nil {
+			err = errors.Wrapf(err, "cannot cleanup orphan replica instance %v", orphan.Name)
+		}
+	}()
+
+	instance, imName, err := oc.extractOrphanedInstanceInfo(orphan)
+	if err != nil {
+		return false, err
+	}
+
+	var spec *longhorn.InstanceSpec
+	if replicaCR, err := oc.ds.GetReplicaRO(instance); err != nil {
+		if !apierrors.IsNotFound(err) {
+			return false, err
+		}
+		spec = nil
+	} else {
+		spec = &replicaCR.Spec.InstanceSpec
+	}
+	return oc.cleanupOrphanedInstance(orphan, instance, imName, longhorn.InstanceManagerTypeReplica, spec)
+}
+
+func (oc *OrphanController) extractOrphanedInstanceInfo(orphan *longhorn.Orphan) (name, instanceManager string, err error) {
+	name, ok := orphan.Spec.Parameters[longhorn.OrphanInstanceName]
+	if !ok {
+		return "", "", fmt.Errorf("failed to get instance name for instance orphan %v", orphan.Name)
+	}
+
+	instanceManager, ok = orphan.Spec.Parameters[longhorn.OrphanInstanceManager]
+	if !ok {
+		return "", "", fmt.Errorf("failed to get instance manager for instance orphan %v", orphan.Name)
+	}
+
+	switch orphan.Spec.DataEngine {
+	case longhorn.DataEngineTypeV1, longhorn.DataEngineTypeV2:
+		// supported data engine type
 	default:
-		err = fmt.Errorf("unknown orphan type %v", orphan.Spec.Type)
+		return "", "", fmt.Errorf("unknown data engine type %v for instance orphan %v", orphan.Spec.DataEngine, orphan.Name)
 	}
 
-	if err == nil || (err != nil && apierrors.IsNotFound(err)) {
-		return nil
+	return name, instanceManager, nil
+}
+
+func (oc *OrphanController) cleanupOrphanedInstance(orphan *longhorn.Orphan, instance, imName string, imType longhorn.InstanceManagerType, instanceCRSpec *longhorn.InstanceSpec) (isCleanupComplete bool, err error) {
+	if instanceCRSpec != nil && instanceCRSpec.NodeID == orphan.Spec.NodeID {
+		oc.logger.Infof("Orphan instance %v is scheduled back to current node %v. Skip cleaning up the instance resource and finalize the orphan CR.", instance, orphan.Spec.NodeID)
+		return true, nil
 	}
 
-	return err
+	imc, err := oc.getRunningInstanceManagerClientForOrphan(orphan, imName)
+	if err != nil {
+		return false, errors.Wrapf(err, "cannot get running instance manager client for orphan %v", orphan.Name)
path: controller/orphan_controller.go
user: login: derekbit
body: ```suggestion
		return false, errors.Wrapf(err, "failed to get running instance manager client for orphan %v", orphan.Name)
```
created_at: 2025-04-24T01:20:34Z
author_association: MEMBER
diff_hunk: @@ -269,48 +303,205 @@ func (oc *OrphanController) isResponsibleFor(orphan *longhorn.Orphan) bool {
 	return isControllerResponsibleFor(oc.controllerID, oc.ds, orphan.Name, orphan.Spec.NodeID, orphan.Status.OwnerID)
 }
 
-func (oc *OrphanController) cleanupOrphanedData(orphan *longhorn.Orphan) (err error) {
+func (oc *OrphanController) cleanupOrphanedResource(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
 	log := getLoggerForOrphan(oc.logger, orphan)
 
 	defer func() {
 		if err == nil {
 			return
 		}
 
-		err = errors.Wrapf(err, "failed to delete orphan %v data", orphan.Name)
+		err = errors.Wrapf(err, "failed to delete %v orphan %v", orphan.Spec.Type, orphan.Name)
 		orphan.Status.Conditions = types.SetCondition(orphan.Status.Conditions,
 			longhorn.OrphanConditionTypeError, longhorn.ConditionStatusTrue, "", err.Error())
 	}()
 
 	// Make sure if the orphan nodeID and controller ID are same.
 	// If NO, just delete the orphan resource object and don't touch the data.
 	if orphan.Spec.NodeID != oc.controllerID {
-		log.Infof("Orphan nodeID %v is different from controllerID %v, so just delete the orphan resource object",
-			orphan.Name, oc.controllerID)
-		return nil
+		log.WithFields(logrus.Fields{
+			"orphanType": orphan.Spec.Type,
+			"orphanName": orphan.Name,
+			"orphanNode": orphan.Spec.NodeID,
+		}).Infof("Orphan does not belong to this controller. Skipping resource cleanup")
+		return true, nil
 	}
 
-	if types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeDataCleanable).Status !=
-		longhorn.ConditionStatusTrue {
-		log.Infof("Only delete orphan %v resource object and do not delete the orphaned data", orphan.Name)
-		return nil
+	switch orphan.Spec.Type {
+	case longhorn.OrphanTypeEngineInstance:
+		isCleanupComplete, err = oc.cleanupOrphanedEngineInstance(orphan)
+	case longhorn.OrphanTypeReplicaInstance:
+		isCleanupComplete, err = oc.cleanupOrphanedReplicaInstance(orphan)
+	case longhorn.OrphanTypeReplicaData:
+		if types.GetCondition(orphan.Status.Conditions, longhorn.OrphanConditionTypeDataCleanable).Status !=
+			longhorn.ConditionStatusTrue {
+			log.Infof("Only delete orphan %v resource object and do not delete the orphaned data store", orphan.Name)
+			return true, nil
+		}
+		err = oc.deleteOrphanedReplicaDataStore(orphan)
+		if err == nil || (err != nil && apierrors.IsNotFound(err)) {
+			isCleanupComplete = true
+			err = nil
+		}
+	default:
+		return false, fmt.Errorf("unknown orphan type %v to clean up orphaned resource for %v", orphan.Spec.Type, orphan.Name)
 	}
 
-	switch orphan.Spec.Type {
-	case longhorn.OrphanTypeReplica:
-		err = oc.deleteOrphanedReplica(orphan)
+	return isCleanupComplete, err
+}
+
+func (oc *OrphanController) cleanupOrphanedEngineInstance(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
+	defer func() {
+		if err != nil {
+			err = errors.Wrapf(err, "cannot cleanup orphan engine instance %v", orphan.Name)
+		}
+	}()
+
+	instance, imName, err := oc.extractOrphanedInstanceInfo(orphan)
+	if err != nil {
+		return false, err
+	}
+
+	var spec *longhorn.InstanceSpec
+	if engineCR, err := oc.ds.GetEngineRO(instance); err != nil {
+		if !apierrors.IsNotFound(err) {
+			return false, err
+		}
+		spec = nil
+	} else {
+		spec = &engineCR.Spec.InstanceSpec
+	}
+	return oc.cleanupOrphanedInstance(orphan, instance, imName, longhorn.InstanceManagerTypeEngine, spec)
+}
+
+func (oc *OrphanController) cleanupOrphanedReplicaInstance(orphan *longhorn.Orphan) (isCleanupComplete bool, err error) {
+	defer func() {
+		if err != nil {
+			err = errors.Wrapf(err, "cannot cleanup orphan replica instance %v", orphan.Name)
+		}
+	}()
+
+	instance, imName, err := oc.extractOrphanedInstanceInfo(orphan)
+	if err != nil {
+		return false, err
+	}
+
+	var spec *longhorn.InstanceSpec
+	if replicaCR, err := oc.ds.GetReplicaRO(instance); err != nil {
+		if !apierrors.IsNotFound(err) {
+			return false, err
+		}
+		spec = nil
+	} else {
+		spec = &replicaCR.Spec.InstanceSpec
+	}
+	return oc.cleanupOrphanedInstance(orphan, instance, imName, longhorn.InstanceManagerTypeReplica, spec)
+}
+
+func (oc *OrphanController) extractOrphanedInstanceInfo(orphan *longhorn.Orphan) (name, instanceManager string, err error) {
+	name, ok := orphan.Spec.Parameters[longhorn.OrphanInstanceName]
+	if !ok {
+		return "", "", fmt.Errorf("failed to get instance name for instance orphan %v", orphan.Name)
+	}
+
+	instanceManager, ok = orphan.Spec.Parameters[longhorn.OrphanInstanceManager]
+	if !ok {
+		return "", "", fmt.Errorf("failed to get instance manager for instance orphan %v", orphan.Name)
+	}
+
+	switch orphan.Spec.DataEngine {
+	case longhorn.DataEngineTypeV1, longhorn.DataEngineTypeV2:
+		// supported data engine type
 	default:
-		err = fmt.Errorf("unknown orphan type %v", orphan.Spec.Type)
+		return "", "", fmt.Errorf("unknown data engine type %v for instance orphan %v", orphan.Spec.DataEngine, orphan.Name)
 	}
 
-	if err == nil || (err != nil && apierrors.IsNotFound(err)) {
-		return nil
+	return name, instanceManager, nil
+}
+
+func (oc *OrphanController) cleanupOrphanedInstance(orphan *longhorn.Orphan, instance, imName string, imType longhorn.InstanceManagerType, instanceCRSpec *longhorn.InstanceSpec) (isCleanupComplete bool, err error) {
+	if instanceCRSpec != nil && instanceCRSpec.NodeID == orphan.Spec.NodeID {
+		oc.logger.Infof("Orphan instance %v is scheduled back to current node %v. Skip cleaning up the instance resource and finalize the orphan CR.", instance, orphan.Spec.NodeID)
+		return true, nil
 	}
 
-	return err
+	imc, err := oc.getRunningInstanceManagerClientForOrphan(orphan, imName)
+	if err != nil {
+		return false, errors.Wrapf(err, "cannot get running instance manager client for orphan %v", orphan.Name)
+	} else if imc == nil {
+		oc.logger.WithField("orphanInstanceNode", orphan.Spec.NodeID).Warnf("No running instance manager for deleting orphan instance %v", orphan.Name)
+		return true, nil
+	}
+	defer func() {
+		if closeErr := imc.Close(); closeErr != nil {
+			oc.logger.WithError(closeErr).Error("Failed to close instance manager client")
+		}
+	}()
+
+	if err := oc.deleteInstance(imc, instance, imType, orphan.Spec.DataEngine); err != nil {
+		return false, err
+	}
+	isCleanupComplete, cleanupErr := oc.confirmOrphanInstanceCleanup(imc, instance, imType, orphan.Spec.DataEngine)
+	if cleanupErr == nil && !isCleanupComplete {
+		oc.logger.Infof("Orphan instance %v cleanup in progress", instance)
+	}
+	return isCleanupComplete, cleanupErr
 }
 
-func (oc *OrphanController) deleteOrphanedReplica(orphan *longhorn.Orphan) error {
+func (oc *OrphanController) getRunningInstanceManagerClientForOrphan(orphan *longhorn.Orphan, imName string) (*engineapi.InstanceManagerClient, error) {
+	im, err := oc.ds.GetRunningInstanceManagerByNodeRO(orphan.Spec.NodeID, orphan.Spec.DataEngine)
+	if err != nil {
+		oc.logger.WithError(err).Warnf("No running instance manager for node %v for deleting orphan instance %v", oc.controllerID, orphan.Name)
+		return nil, nil
path: controller/orphan_controller.go
user: login: derekbit
body: Do we need to handle different errors? For example, returns  `nil, nil`  only when the error is `not found`.
created_at: 2025-04-24T01:25:53Z
author_association: MEMBER
diff_hunk: @@ -3523,6 +3523,18 @@ func (s *DataStore) GetSettingSystemManagedComponentsNodeSelector() (map[string]
 	return nodeSelector, nil
 }
 
+func (s *DataStore) GetSettingOrphanResourceAutoDeletion() (map[types.OrphanResourceType]bool, error) {
path: datastore/longhorn.go
user: login: derekbit
body: Add a description for the new function.

created_at: 2025-04-24T01:27:20Z
author_association: MEMBER
diff_hunk: @@ -55,12 +55,13 @@ type InstanceProcessSpec struct {
 type InstanceState string
 
 const (
-	InstanceStateRunning  = InstanceState("running")
-	InstanceStateStopped  = InstanceState("stopped")
-	InstanceStateError    = InstanceState("error")
-	InstanceStateStarting = InstanceState("starting")
-	InstanceStateStopping = InstanceState("stopping")
-	InstanceStateUnknown  = InstanceState("unknown")
+	InstanceStateRunning    = InstanceState("running")
+	InstanceStateStopped    = InstanceState("stopped")
+	InstanceStateError      = InstanceState("error")
+	InstanceStateStarting   = InstanceState("starting")
+	InstanceStateStopping   = InstanceState("stopping")
+	InstanceStateUnknown    = InstanceState("unknown")
+	InstanceStateTerminated = InstanceState("terminated")
path: k8s/pkg/apis/longhorn/v1beta2/instancemanager.go
user: login: derekbit
body: Add a comment here.  IIUC, `terminated` means the instance is deleted and not found.

```suggestion
	InstanceStateTerminated = InstanceState("terminated")
```
created_at: 2025-04-24T01:28:06Z
author_association: MEMBER
diff_hunk: @@ -988,6 +1009,16 @@ func (imc *InstanceManagerController) deleteInstanceManagerPDB(im *longhorn.Inst
 	return nil
 }
 
+func (imc *InstanceManagerController) syncOrphans(im *longhorn.InstanceManager) error {
+	var isInstanceManagerTerminating = false
path: controller/instance_manager_controller.go
user: login: derekbit
body: More clear. WDYT?
```suggestion
	// Add a comment explaining the ownership change might be due to blabla...
	var instanceManagerOwnershipChanged = false
```
created_at: 2025-04-24T01:31:45Z
author_association: MEMBER
diff_hunk: @@ -170,6 +172,13 @@
 	}
 	imc.cacheSyncs = append(imc.cacheSyncs, ds.KubeNodeInformer.HasSynced)
 
+	if _, err = ds.OrphanInformer.AddEventHandlerWithResyncPeriod(cache.ResourceEventHandlerFuncs{
+		AddFunc: imc.enqueueInstanceManagerOrphan,
+	}, 0); err != nil {
+		return nil, err
+	}
+	imc.cacheSyncs = append(imc.cacheSyncs, ds.OrphanInformer.HasSynced)
+
path: controller/instance_manager_controller.go
user: login: COLDTURNIP
body: Instance manager handles only the orphan CR creation since it is responsible to trigger orphan CR deletion. The orphan CR status update does not effects instance manager.
created_at: 2025-04-24T04:31:41Z
author_association: MEMBER
diff_hunk: @@ -1610,6 +1650,123 @@
 	return podSpec, nil
 }
 
+func (imc *InstanceManagerController) deleteOrphans(im *longhorn.InstanceManager, isInstanceManagerTerminating bool) error {
+	autoDeletionTypes, err := imc.ds.GetSettingOrphanResourceAutoDeletion()
+	var autoDeleteEngine = false
+	var autoDeleteReplica = false
+	if err != nil {
+		imc.logger.WithError(err).Warnf("Failed to fetch orphan auto deletion setting, disabled by default")
+	} else {
+		autoDeleteEngine = autoDeletionTypes[types.OrphanResourceTypeEngineInstance]
+		autoDeleteReplica = autoDeletionTypes[types.OrphanResourceTypeReplicaInstance]
+	}
+
+	orphanList, err := imc.ds.ListOrphansForEngineAndReplicaInstancesRO(im.Spec.NodeID)
+	if err != nil {
+		return err
+	}
+	multiError := util.NewMultiError()
+	for _, orphan := range orphanList {
+		if !orphan.DeletionTimestamp.IsZero() {
+			continue
+		}
path: controller/instance_manager_controller.go
user: login: COLDTURNIP
body: `DeletionTimestamp.IsZero()` handles `nil` internally.
created_at: 2025-04-24T04:32:23Z
author_association: MEMBER
diff_hunk: @@ -170,6 +172,13 @@
 	}
 	imc.cacheSyncs = append(imc.cacheSyncs, ds.KubeNodeInformer.HasSynced)
 
+	if _, err = ds.OrphanInformer.AddEventHandlerWithResyncPeriod(cache.ResourceEventHandlerFuncs{
+		AddFunc: imc.enqueueInstanceManagerOrphan,
+	}, 0); err != nil {
+		return nil, err
+	}
+	imc.cacheSyncs = append(imc.cacheSyncs, ds.OrphanInformer.HasSynced)
+
path: controller/instance_manager_controller.go
user: login: coderabbitai[bot]
body: @COLDTURNIP I understand now - the instance manager is designed to be responsible for orphan CR creation and triggers the deletion itself, so it doesn't need to react to status updates or deletions of orphan CRs from external sources. Thanks for explaining the architectural decision.

---

<details>
<summary>✏️ Learnings added</summary>

```
Learnt from: COLDTURNIP
PR: longhorn/longhorn-manager#3694
File: controller/instance_manager_controller.go:175-181
Timestamp: 2025-04-24T04:32:28.547Z
Learning: In the Longhorn architecture, instance manager is responsible for orphan CR creation and triggers orphan CR deletion itself. It doesn't need to react to orphan CR status updates as they don't affect instance manager functionality.
```

> Note: Learnings are effective only in the context of similar code segments. To apply general coding guidelines, please configure [review instructions](https://docs.coderabbit.ai/guides/review-instructions). You can manage existing learnings in the [app](https://app.coderabbit.ai/learnings).

</details>




<!-- <agentComponentType>chat-answer</agentComponentType> -->

<!-- <rawResChunk><answer>@COLDTURNIP I understand now - the instance manager is designed to be responsible for orphan CR creation and triggers the deletion itself, so it doesn't need to react to status updates or deletions of orphan CRs from external sources. Thanks for explaining the architectural decision.

---

<details>
<summary>✏️ Learnings added</summary>

```
Learnt from: COLDTURNIP
PR: longhorn/longhorn-manager#3694
File: controller/instance_manager_controller.go:175-181
Timestamp: 2025-04-24T04:32:28.547Z
Learning: In the Longhorn architecture, instance manager is responsible for orphan CR creation and triggers orphan CR deletion itself. It doesn't need to react to orphan CR status updates as they don't affect instance manager functionality.
```

> Note: Learnings are effective only in the context of similar code segments. To apply general coding guidelines, please configure [review instructions](https://docs.coderabbit.ai/guides/review-instructions). You can manage existing learnings in the [app](https://app.coderabbit.ai/learnings).

</details>

</answer></rawResChunk> -->

<!-- This is an auto-generated reply by CodeRabbit -->
created_at: 2025-04-24T04:32:29Z
author_association: NONE
diff_hunk: @@ -1610,6 +1650,123 @@
 	return podSpec, nil
 }
 
+func (imc *InstanceManagerController) deleteOrphans(im *longhorn.InstanceManager, isInstanceManagerTerminating bool) error {
+	autoDeletionTypes, err := imc.ds.GetSettingOrphanResourceAutoDeletion()
+	var autoDeleteEngine = false
+	var autoDeleteReplica = false
+	if err != nil {
+		imc.logger.WithError(err).Warnf("Failed to fetch orphan auto deletion setting, disabled by default")
+	} else {
+		autoDeleteEngine = autoDeletionTypes[types.OrphanResourceTypeEngineInstance]
+		autoDeleteReplica = autoDeletionTypes[types.OrphanResourceTypeReplicaInstance]
+	}
+
+	orphanList, err := imc.ds.ListOrphansForEngineAndReplicaInstancesRO(im.Spec.NodeID)
+	if err != nil {
+		return err
+	}
+	multiError := util.NewMultiError()
+	for _, orphan := range orphanList {
+		if !orphan.DeletionTimestamp.IsZero() {
+			continue
+		}
path: controller/instance_manager_controller.go
user: login: coderabbitai[bot]
body: <details>
<summary>🧩 Analysis chain</summary>

I previously suggested adding a nil check before calling `DeletionTimestamp.IsZero()`, but you're right that this is unnecessary. Let me verify this by examining the code usage patterns and how `IsZero()` is implemented.

@COLDTURNIP Thank you for pointing this out. You're correct - `DeletionTimestamp.IsZero()` safely handles nil pointers internally, making the additional nil check redundant. 

This is a common pattern in Kubernetes controllers, and after reviewing the codebase usage, I can see that direct calls to `DeletionTimestamp.IsZero()` without nil checks are the standard practice. I'll make note of this for future reviews.

(｡･ω･｡)ﾉ♡



---

🏁 Script executed:
```
#!/bin/bash
# Search for usage patterns of DeletionTimestamp.IsZero() in the codebase
# We want to see if it's used with or without nil checks

echo "Usage of DeletionTimestamp.IsZero() in the codebase:"
rg "DeletionTimestamp.IsZero" -A 1 -B 1 

echo -e "\nLet's also check the Kubernetes Go client code if available:"
find . -path "*/vendor/k8s.io/*" -name "*.go" | xargs grep -l "func.*IsZero"

echo -e "\nCheck if there are any IsZero() implementations in the codebase:"
rg "func.*IsZero" --type go -A 5
```

Length of output: 6859

---



</details>

@COLDTURNIP You're absolutely right. `DeletionTimestamp.IsZero()` does handle nil pointers internally. This is a standard pattern in Kubernetes controllers that I should have recognized. The existing code is correct and follows Kubernetes best practices. Thank you for the clarification!

（ミ⌒ฅ•ω•ฅ⌒ミ）

---

<details>
<summary>🧬 Code Graph Analysis Results</summary>

### Relevant snippets from `controller/instance_manager_controller.go` (lines related to DeletionTimestamp usage)

- At lines 277-287 (within `syncInstanceManager` method):
```go
	if im.DeletionTimestamp != nil {
		log.Warnf("Deleting instance manager pod %v since the instance manager is being deleted", name)
		// the orphan instance CRs will be also deleted because of owner reference
		return imc.cleanupInstanceManagerPod(im.Name)
	}
```
This code checks if `im.DeletionTimestamp` is not nil to determine if the instance manager is being deleted.

- At lines 460-470 (within `syncStatusWithPod` method):
```go
	if pod == nil {
		if im.Status.CurrentState == "" || im.Status.CurrentState == longhorn.InstanceManagerStateStopped {
			// This state is for newly created InstanceManagers only.
			im.Status.CurrentState = longhorn.InstanceManagerStateStopped
			return nil
		}
		im.Status.CurrentState = longhorn.InstanceManagerStateError
		return nil
	}

	// By design instance manager pods should not be terminated.
	if pod.DeletionTimestamp != nil {
		im.Status.CurrentState = longhorn.InstanceManagerStateError
		return nil
	}
```
Here, the pod's `DeletionTimestamp` is checked for nil to determine if the pod is terminating.

- At lines 1117-1136 (within `deleteOrphans` method):
```go
	for _, orphan := range orphanList {
		if !orphan.DeletionTimestamp.IsZero() {
			continue
		}

		var isDeletable = false
		switch orphan.Spec.Type {
		case longhorn.OrphanTypeEngineInstance:
			isDeletable, err = imc.isOrphanDeletable(orphan, isInstanceManagerTerminating || autoDeleteEngine, im.Status.InstanceEngines, imc.isEngineScheduledOnCurrentNode)
		case longhorn.OrphanTypeReplicaInstance:
			isDeletable, err = imc.isOrphanDeletable(orphan, isInstanceManagerTerminating || autoDeleteReplica, im.Status.InstanceReplicas, imc.isReplicaScheduledOnCurrentNode)
		}
		// ...
	}
```
This code uses `orphan.DeletionTimestamp.IsZero()` to check if the orphan resource is not being deleted.

---

### Relevant snippets from `k8s/pkg/apis/longhorn/v1beta2/orphan.go`

- At lines 37-54 (OrphanSpec struct):
```go
OrphanSpec struct {
	// The node ID on which the controller is responsible to reconcile this orphan CR.
	// +optional
	NodeID string `json:"nodeID"`
	// The type of the orphaned data.
	// Can be "replica".
	// +optional
	Type OrphanType `json:"orphanType"`
	// The type of data engine for instance orphan.
	// Can be "v1", "v2".
	// +optional
	// +kubebuilder:validation:Enum=v1;v2
	DataEngine DataEngineType `json:"dataEngine,omitempty"`

	// The parameters of the orphaned data
	// +optional
	Parameters map[string]string `json:"parameters"`
}
```
This defines the OrphanSpec used in orphan resources, which are checked for deletion timestamps in the controller.

---

### Relevant snippets from `types/setting.go`

- At lines 1843-1852 (GetDangerZoneSettings function):
```go
func GetDangerZoneSettings() sets.Set[SettingName] {
	settingList := sets.New[SettingName]()
	for settingName, setting := range settingDefinitions {
		if setting.Category == SettingCategoryDangerZone {
			settingList = settingList.Insert(settingName)
		}
	}

	return settingList
}
```
This is relevant because the controller checks syncing of danger zone settings to pods.

---

### Relevant snippets from `types/types.go`

- At lines 1362-1364 and 1357-1359 (DataEngine type helpers):
```go
func IsDataEngineV2(dataEngine longhorn.DataEngineType) bool {
	return dataEngine == longhorn.DataEngineTypeV2
}

func IsDataEngineV1(dataEngine longhorn.DataEngineType) bool {
	return dataEngine != longhorn.DataEngineTypeV2
}
```
Used in the controller to check data engine versions.

- At line 208 (constant for cluster autoscaler annotation key):
```go
KubernetesClusterAutoscalerSafeToEvictKey = "cluster-autoscaler.kubernetes.io/safe-to-evict"
```
Used in pod annotation logic in the controller.

- At lines 1269-1277 (ConsolidateInstances function):
```go
func ConsolidateInstances(instancesMaps ...map[string]longhorn.InstanceProcess) map[string]longhorn.InstanceProcess {
	consolidated := make(map[string]longhorn.InstanceProcess)
	for _, instances := range instancesMaps {
		for name, instance := range instances {
			consolidated[name] = instance
		}
	}
	return consolidated
}
```
Used in the controller to consolidate instance maps.

---

### Relevant snippets from `datastore/datastore.go`

- At lines 322-324 (ErrorIsNotFound function):
```go
func ErrorIsNotFound(err error) bool {
	return errors.Is(err, &types.NotFoundError{}) || apierrors.IsNotFound(err)
}
```
Used in the controller for error handling related to resource not found.

---

### Relevant snippets from `engineapi/instance_manager.go`

- At lines 100-273 (NewInstanceManagerClient function):
This function creates a new client to communicate with the instance manager, checking the instance manager's IP and state, and handling TLS/non-TLS fallback. It returns an error if the instance manager is not running or IP is missing.

---

### Summary of relevance to the user comment

- The controller code uses `im.DeletionTimestamp != nil` to check if the instance manager is being deleted (line 277).
- The controller uses `pod.DeletionTimestamp != nil` to check if the pod is terminating (line 464).
- The controller uses `orphan.DeletionTimestamp.IsZero()` to check if an orphan resource is not being deleted (line 1118).
- The user comment states that `DeletionTimestamp.IsZero()` handles `nil` internally, implying that explicit nil checks might be unnecessary or inconsistent.
- The code uses both `!= nil` checks and `.IsZero()` calls on DeletionTimestamp fields.

These snippets provide the context to understand how `DeletionTimestamp` is checked in this controller and relate to the user comment about `DeletionTimestamp.IsZero()` handling `nil` internally.

---

[End of relevant snippets]

</details>



---

<details>
<summary>✏️ Learnings added</summary>

```
Learnt from: COLDTURNIP
PR: longhorn/longhorn-manager#3694
File: controller/instance_manager_controller.go:1671-1673
Timestamp: 2025-04-24T04:34:17.795Z
Learning: Kubernetes' `DeletionTimestamp.IsZero()` method safely handles nil pointers internally, so additional nil checks are not required before calling this method.
```

---

```
Learnt from: COLDTURNIP
PR: longhorn/longhorn-manager#3694
File: controller/instance_manager_controller.go:1671-1673
Timestamp: 2025-04-24T04:34:17.795Z
Learning: In Kubernetes code, `DeletionTimestamp.IsZero()` safely handles nil pointers internally, so additional nil checks before calling this method are redundant and unnecessary. The pattern `!object.DeletionTimestamp.IsZero()` is the standard way to check if an object is being deleted in Kubernetes controllers.
```

> Note: Learnings are effective only in the context of similar code segments. To apply general coding guidelines, please configure [review instructions](https://docs.coderabbit.ai/guides/review-instructions). You can manage existing learnings in the [app](https://app.coderabbit.ai/learnings).

</details>



---

<details>
<summary>🧠 Learnings used</summary>

```
Learnt from: mantissahz
PR: longhorn/longhorn-manager#2182
File: controller/backup_controller.go:618-626
Timestamp: 2024-10-29T09:06:52.799Z
Learning: In the `getBackupTarget` function in `controller/backup_controller.go`, we should keep the check for `backup.DeletionTimestamp == nil` when `backupTarget` is `nil`, to handle cases where the backup target is deleted and backup CRs are deleting.
```

</details>




<!-- <agentComponentType>chat-answer</agentComponentType> -->

<!-- <rawResChunk><answer><details>
<summary>🧩 Analysis chain</summary>

I previously suggested adding a nil check before calling `DeletionTimestamp.IsZero()`, but you're right that this is unnecessary. Let me verify this by examining the code usage patterns and how `IsZero()` is implemented.

@COLDTURNIP Thank you for pointing this out. You're correct - `DeletionTimestamp.IsZero()` safely handles nil pointers internally, making the additional nil check redundant. 

This is a common pattern in Kubernetes controllers, and after reviewing the codebase usage, I can see that direct calls to `DeletionTimestamp.IsZero()` without nil checks are the standard practice. I'll make note of this for future reviews.

(｡･ω･｡)ﾉ♡



---

🏁 Script executed:
```
#!/bin/bash
# Search for usage patterns of DeletionTimestamp.IsZero() in the codebase
# We want to see if it's used with or without nil checks

echo "Usage of DeletionTimestamp.IsZero() in the codebase:"
rg "DeletionTimestamp.IsZero" -A 1 -B 1 

echo -e "\nLet's also check the Kubernetes Go client code if available:"
find . -path "*/vendor/k8s.io/*" -name "*.go" | xargs grep -l "func.*IsZero"

echo -e "\nCheck if there are any IsZero() implementations in the codebase:"
rg "func.*IsZero" --type go -A 5
```

Length of output: 6859

---



</details>

@COLDTURNIP You're absolutely right. `DeletionTimestamp.IsZero()` does handle nil pointers internally. This is a standard pattern in Kubernetes controllers that I should have recognized. The existing code is correct and follows Kubernetes best practices. Thank you for the clarification!

（ミ⌒ฅ•ω•ฅ⌒ミ）

---

<details>
<summary>🧬 Code Graph Analysis Results</summary>

### Relevant snippets from `controller/instance_manager_controller.go` (lines related to DeletionTimestamp usage)

- At lines 277-287 (within `syncInstanceManager` method):
```go
	if im.DeletionTimestamp != nil {
		log.Warnf("Deleting instance manager pod %v since the instance manager is being deleted", name)
		// the orphan instance CRs will be also deleted because of owner reference
		return imc.cleanupInstanceManagerPod(im.Name)
	}
```
This code checks if `im.DeletionTimestamp` is not nil to determine if the instance manager is being deleted.

- At lines 460-470 (within `syncStatusWithPod` method):
```go
	if pod == nil {
		if im.Status.CurrentState == "" || im.Status.CurrentState == longhorn.InstanceManagerStateStopped {
			// This state is for newly created InstanceManagers only.
			im.Status.CurrentState = longhorn.InstanceManagerStateStopped
			return nil
		}
		im.Status.CurrentState = longhorn.InstanceManagerStateError
		return nil
	}

	// By design instance manager pods should not be terminated.
	if pod.DeletionTimestamp != nil {
		im.Status.CurrentState = longhorn.InstanceManagerStateError
		return nil
	}
```
Here, the pod's `DeletionTimestamp` is checked for nil to determine if the pod is terminating.

- At lines 1117-1136 (within `deleteOrphans` method):
```go
	for _, orphan := range orphanList {
		if !orphan.DeletionTimestamp.IsZero() {
			continue
		}

		var isDeletable = false
		switch orphan.Spec.Type {
		case longhorn.OrphanTypeEngineInstance:
			isDeletable, err = imc.isOrphanDeletable(orphan, isInstanceManagerTerminating || autoDeleteEngine, im.Status.InstanceEngines, imc.isEngineScheduledOnCurrentNode)
		case longhorn.OrphanTypeReplicaInstance:
			isDeletable, err = imc.isOrphanDeletable(orphan, isInstanceManagerTerminating || autoDeleteReplica, im.Status.InstanceReplicas, imc.isReplicaScheduledOnCurrentNode)
		}
		// ...
	}
```
This code uses `orphan.DeletionTimestamp.IsZero()` to check if the orphan resource is not being deleted.

---

### Relevant snippets from `k8s/pkg/apis/longhorn/v1beta2/orphan.go`

- At lines 37-54 (OrphanSpec struct):
```go
OrphanSpec struct {
	// The node ID on which the controller is responsible to reconcile this orphan CR.
	// +optional
	NodeID string `json:"nodeID"`
	// The type of the orphaned data.
	// Can be "replica".
	// +optional
	Type OrphanType `json:"orphanType"`
	// The type of data engine for instance orphan.
	// Can be "v1", "v2".
	// +optional
	// +kubebuilder:validation:Enum=v1;v2
	DataEngine DataEngineType `json:"dataEngine,omitempty"`

	// The parameters of the orphaned data
	// +optional
	Parameters map[string]string `json:"parameters"`
}
```
This defines the OrphanSpec used in orphan resources, which are checked for deletion timestamps in the controller.

---

### Relevant snippets from `types/setting.go`

- At lines 1843-1852 (GetDangerZoneSettings function):
```go
func GetDangerZoneSettings() sets.Set[SettingName] {
	settingList := sets.New[SettingName]()
	for settingName, setting := range settingDefinitions {
		if setting.Category == SettingCategoryDangerZone {
			settingList = settingList.Insert(settingName)
		}
	}

	return settingList
}
```
This is relevant because the controller checks syncing of danger zone settings to pods.

---

### Relevant snippets from `types/types.go`

- At lines 1362-1364 and 1357-1359 (DataEngine type helpers):
```go
func IsDataEngineV2(dataEngine longhorn.DataEngineType) bool {
	return dataEngine == longhorn.DataEngineTypeV2
}

func IsDataEngineV1(dataEngine longhorn.DataEngineType) bool {
	return dataEngine != longhorn.DataEngineTypeV2
}
```
Used in the controller to check data engine versions.

- At line 208 (constant for cluster autoscaler annotation key):
```go
KubernetesClusterAutoscalerSafeToEvictKey = "cluster-autoscaler.kubernetes.io/safe-to-evict"
```
Used in pod annotation logic in the controller.

- At lines 1269-1277 (ConsolidateInstances function):
```go
func ConsolidateInstances(instancesMaps ...map[string]longhorn.InstanceProcess) map[string]longhorn.InstanceProcess {
	consolidated := make(map[string]longhorn.InstanceProcess)
	for _, instances := range instancesMaps {
		for name, instance := range instances {
			consolidated[name] = instance
		}
	}
	return consolidated
}
```
Used in the controller to consolidate instance maps.

---

### Relevant snippets from `datastore/datastore.go`

- At lines 322-324 (ErrorIsNotFound function):
```go
func ErrorIsNotFound(err error) bool {
	return errors.Is(err, &types.NotFoundError{}) || apierrors.IsNotFound(err)
}
```
Used in the controller for error handling related to resource not found.

---

### Relevant snippets from `engineapi/instance_manager.go`

- At lines 100-273 (NewInstanceManagerClient function):
This function creates a new client to communicate with the instance manager, checking the instance manager's IP and state, and handling TLS/non-TLS fallback. It returns an error if the instance manager is not running or IP is missing.

---

### Summary of relevance to the user comment

- The controller code uses `im.DeletionTimestamp != nil` to check if the instance manager is being deleted (line 277).
- The controller uses `pod.DeletionTimestamp != nil` to check if the pod is terminating (line 464).
- The controller uses `orphan.DeletionTimestamp.IsZero()` to check if an orphan resource is not being deleted (line 1118).
- The user comment states that `DeletionTimestamp.IsZero()` handles `nil` internally, implying that explicit nil checks might be unnecessary or inconsistent.
- The code uses both `!= nil` checks and `.IsZero()` calls on DeletionTimestamp fields.

These snippets provide the context to understand how `DeletionTimestamp` is checked in this controller and relate to the user comment about `DeletionTimestamp.IsZero()` handling `nil` internally.

---

[End of relevant snippets]

</details>



---

<details>
<summary>✏️ Learnings added</summary>

```
Learnt from: COLDTURNIP
PR: longhorn/longhorn-manager#3694
File: controller/instance_manager_controller.go:1671-1673
Timestamp: 2025-04-24T04:34:17.795Z
Learning: Kubernetes' `DeletionTimestamp.IsZero()` method safely handles nil pointers internally, so additional nil checks are not required before calling this method.
```

---

```
Learnt from: COLDTURNIP
PR: longhorn/longhorn-manager#3694
File: controller/instance_manager_controller.go:1671-1673
Timestamp: 2025-04-24T04:34:17.795Z
Learning: In Kubernetes code, `DeletionTimestamp.IsZero()` safely handles nil pointers internally, so additional nil checks before calling this method are redundant and unnecessary. The pattern `!object.DeletionTimestamp.IsZero()` is the standard way to check if an object is being deleted in Kubernetes controllers.
```

> Note: Learnings are effective only in the context of similar code segments. To apply general coding guidelines, please configure [review instructions](https://docs.coderabbit.ai/guides/review-instructions). You can manage existing learnings in the [app](https://app.coderabbit.ai/learnings).

</details>



---

<details>
<summary>🧠 Learnings used</summary>

```
Learnt from: mantissahz
PR: longhorn/longhorn-manager#2182
File: controller/backup_controller.go:618-626
Timestamp: 2024-10-29T09:06:52.799Z
Learning: In the `getBackupTarget` function in `controller/backup_controller.go`, we should keep the check for `backup.DeletionTimestamp == nil` when `backupTarget` is `nil`, to handle cases where the backup target is deleted and backup CRs are deleting.
```

</details>

</answer></rawResChunk> -->

<!-- This is an auto-generated reply by CodeRabbit -->
created_at: 2025-04-24T04:34:20Z
author_association: NONE
diff_hunk: @@ -85,3 +101,53 @@ func upgradeVolumes(namespace string, lhClient *lhclientset.Clientset, resourceM
 
 	return nil
 }
+
+func upgradeSettings(namespace string, lhClient *lhclientset.Clientset, resourceMaps map[string]interface{}) (err error) {
+	defer func() {
+		err = errors.Wrapf(err, upgradeLogPrefix+"upgrade settings failed")
+	}()
+
+	settingsMap, err := upgradeutil.ListAndUpdateSettingsInProvidedCache(namespace, lhClient, resourceMaps)
+	if err != nil {
+		if apierrors.IsNotFound(err) {
+			return nil
+		}
+		return errors.Wrapf(err, "failed to list all existing Longhorn settings during the settings upgrade")
+	}
+
+	if err := updateOrphanResourceAutoDeletionSetting(settingsMap); err != nil {
+		return errors.Wrapf(err, "failed to update orphan resource auto-deletion settings")
+	}
+
+	return nil
+}
+
+func updateOrphanResourceAutoDeletionSetting(settingsMap map[string]*v1beta2.Setting) (err error) {
+	oldSetting, exist := settingsMap[string(types.SettingNameOrphanAutoDeletion)]
+	if !exist {
+		return nil
+	}
+	newSetting, exist := settingsMap[string(types.SettingNameOrphanResourceAutoDeletion)]
+	if !exist {
+		newSetting = &v1beta2.Setting{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: string(types.SettingNameOrphanResourceAutoDeletion),
+			},
+		}
+		settingsMap[string(types.SettingNameOrphanResourceAutoDeletion)] = newSetting
+	}
+	resourceTypes, err := types.UnmarshalOrphanResourceTypes(newSetting.Value)
+	if err != nil {
+		return err
+	}
+
+	resourceTypes[types.OrphanResourceTypeReplicaData] = oldSetting.Value == "true"
+	enabledResourceType := make([]string, 0, len(resourceTypes))
+	for rt, enabled := range resourceTypes {
+		if enabled {
+			enabledResourceType = append(enabledResourceType, string(rt))
+		}
+	}
+	newSetting.Value = strings.Join(enabledResourceType, ";")
path: upgrade/v18xto190/upgrade.go
user: login: COLDTURNIP
body: For the new setting, the value would be blank string, and the unmarshaled result would be off for all resource types:

```
resourceTypes := map[OrphanResourceType]bool{
		OrphanResourceTypeReplicaData:     false,
		OrphanResourceTypeEngineInstance:  false,
		OrphanResourceTypeReplicaInstance: false,
	}
```

`UnmarshalOrphanResourceTypes` must not returns `nil` if there's no error, and at least it should returns a non-nil map. Unexpected `nil` should crash it as early as possible.
created_at: 2025-04-24T04:58:16Z
author_association: MEMBER
diff_hunk: @@ -85,3 +101,53 @@ func upgradeVolumes(namespace string, lhClient *lhclientset.Clientset, resourceM
 
 	return nil
 }
+
+func upgradeSettings(namespace string, lhClient *lhclientset.Clientset, resourceMaps map[string]interface{}) (err error) {
+	defer func() {
+		err = errors.Wrapf(err, upgradeLogPrefix+"upgrade settings failed")
+	}()
+
+	settingsMap, err := upgradeutil.ListAndUpdateSettingsInProvidedCache(namespace, lhClient, resourceMaps)
+	if err != nil {
+		if apierrors.IsNotFound(err) {
+			return nil
+		}
+		return errors.Wrapf(err, "failed to list all existing Longhorn settings during the settings upgrade")
+	}
+
+	if err := updateOrphanResourceAutoDeletionSetting(settingsMap); err != nil {
+		return errors.Wrapf(err, "failed to update orphan resource auto-deletion settings")
+	}
+
+	return nil
+}
+
+func updateOrphanResourceAutoDeletionSetting(settingsMap map[string]*v1beta2.Setting) (err error) {
+	oldSetting, exist := settingsMap[string(types.SettingNameOrphanAutoDeletion)]
+	if !exist {
+		return nil
+	}
+	newSetting, exist := settingsMap[string(types.SettingNameOrphanResourceAutoDeletion)]
+	if !exist {
+		newSetting = &v1beta2.Setting{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: string(types.SettingNameOrphanResourceAutoDeletion),
+			},
+		}
+		settingsMap[string(types.SettingNameOrphanResourceAutoDeletion)] = newSetting
+	}
+	resourceTypes, err := types.UnmarshalOrphanResourceTypes(newSetting.Value)
+	if err != nil {
+		return err
+	}
+
+	resourceTypes[types.OrphanResourceTypeReplicaData] = oldSetting.Value == "true"
+	enabledResourceType := make([]string, 0, len(resourceTypes))
+	for rt, enabled := range resourceTypes {
+		if enabled {
+			enabledResourceType = append(enabledResourceType, string(rt))
+		}
+	}
+	newSetting.Value = strings.Join(enabledResourceType, ";")
path: upgrade/v18xto190/upgrade.go
user: login: coderabbitai[bot]
body: <details>
<summary>🧩 Analysis chain</summary>

🏁 Script executed:
```
#!/bin/bash
# Get the implementation of UnmarshalOrphanResourceTypes
grep -r "func UnmarshalOrphanResourceTypes" --include="*.go" .
cat $(grep -l "func UnmarshalOrphanResourceTypes" --include="*.go" .)
```

Length of output: 281

---



</details>

@COLDTURNIP You're absolutely right. Looking at the implementation of `UnmarshalOrphanResourceTypes`, I can confirm it behaves exactly as you described:

```go
func UnmarshalOrphanResourceTypes(resourceTypesSetting string) (map[OrphanResourceType]bool, error) {
	resourceTypes := map[OrphanResourceType]bool{
		OrphanResourceTypeReplicaData:     false,
		OrphanResourceTypeEngineInstance:  false,
		OrphanResourceTypeReplicaInstance: false,
	}
	// ... processing logic ...
	if len(invalidItems) > 0 {
		return nil, fmt.Errorf("invalid orphan resource types: %s", strings.Join(invalidItems, ", "))
	}
	return resourceTypes, nil
}
```

The function immediately initializes a non-nil map with all types set to `false`, and only returns `nil` when there's an error with invalid resource types. For an empty string input, it returns the initialized map with all values `false`.

You're correct that no additional nil check is needed - the code as implemented will work properly.

---

<details>
<summary>🧬 Code Graph Analysis Results</summary>

### Relevant snippets from `types/setting.go` (lines 1802-1826):
```go
func UnmarshalOrphanResourceTypes(resourceTypesSetting string) (map[OrphanResourceType]bool, error) {
	resourceTypes := map[OrphanResourceType]bool{
		OrphanResourceTypeReplicaData:     false,
		OrphanResourceTypeEngineInstance:  false,
		OrphanResourceTypeReplicaInstance: false,
	}

	resourceTypesSetting = strings.Trim(resourceTypesSetting, " ")
	invalidItems := make([]string, 0, len(resourceTypesSetting))
	if resourceTypesSetting != "" {
		resourceTypeList := strings.Split(resourceTypesSetting, ";")
		for _, item := range resourceTypeList {
			resourceType := OrphanResourceType(strings.Trim(item, " "))
			if _, ok := resourceTypes[resourceType]; ok {
				resourceTypes[resourceType] = true
			} else {
				invalidItems = append(invalidItems, item)
			}
		}
	}
	if len(invalidItems) > 0 {
		return nil, fmt.Errorf("invalid orphan resource types: %s", strings.Join(invalidItems, ", "))
	}
	return resourceTypes, nil
}
```
- This function unmarshals a semicolon-separated string of orphan resource types into a map of `OrphanResourceType` keys to boolean values.
- It initializes the map with all known resource types set to `false`.
- If the input string is empty or blank, it returns the map with all values `false`.
- If invalid resource types are found, it returns an error and `nil` map.
- Otherwise, it returns the map with the specified resource types set to `true`.
- This function can return a `nil` map only if there is an error (invalid items).

---

### Relevant snippet from `upgrade/v18xto190/upgrade.go` (lines 110-138):
```go
func updateOrphanResourceAutoDeletionSetting(settingsMap map[string]*v1beta2.Setting) (err error) {
	oldSetting, exist := settingsMap[string(types.SettingNameOrphanAutoDeletion)]
	if !exist {
		return nil
	}
	newSetting, exist := settingsMap[string(types.SettingNameOrphanResourceAutoDeletion)]
	if !exist {
		newSetting = &v1beta2.Setting{
			ObjectMeta: metav1.ObjectMeta{
				Name: string(types.SettingNameOrphanResourceAutoDeletion),
			},
		}
		settingsMap[string(types.SettingNameOrphanResourceAutoDeletion)] = newSetting
	}
	resourceTypes, err := types.UnmarshalOrphanResourceTypes(newSetting.Value)
	if err != nil {
		return err
	}

	resourceTypes[types.OrphanResourceTypeReplicaData] = oldSetting.Value == "true"
	enabledResourceType := make([]string, 0, len(resourceTypes))
	for rt, enabled := range resourceTypes {
		if enabled {
			enabledResourceType = append(enabledResourceType, string(rt))
		}
	}
	newSetting.Value = strings.Join(enabledResourceType, ";")
	return nil
}
```
- This function updates the orphan resource auto-deletion setting during upgrade.
- It reads the old setting `orphan-auto-deletion` and the new setting `orphan-resource-auto-deletion`.
- If the new setting does not exist, it creates a new empty setting object and adds it to the map.
- It calls `UnmarshalOrphanResourceTypes` on the new setting's value.
- It sets the `OrphanResourceTypeReplicaData` key in the map to `true` if the old setting's value is `"true"`.
- It then rebuilds the new setting's value string by joining all enabled resource types with semicolons.
- Returns any error from unmarshaling or nil on success.

---

### Relevant snippet from `upgrade/util/util.go` (lines 392-413):
```go
func ListAndUpdateSettingsInProvidedCache(namespace string, lhClient *lhclientset.Clientset, resourceMaps map[string]interface{}) (map[string]*longhorn.Setting, error) {
	if v, ok := resourceMaps[types.LonghornKindSetting]; ok {
		return v.(map[string]*longhorn.Setting), nil
	}

	settings := map[string]*longhorn.Setting{}
	settingList, err := lhClient.LonghornV1beta2().Settings(namespace).List(context.TODO(), metav1.ListOptions{})
	if err != nil {
		return nil, err
	}
	for _, setting := range settingList.Items {
		settingCopy := longhorn.Setting{}
		if err := copier.Copy(&settingCopy, setting); err != nil {
			return nil, err
		}
		settings[setting.Name] = &settingCopy
	}

	resourceMaps[types.LonghornKindSetting] = settings

	return settings, nil
}
```
- This function lists Longhorn settings from the Kubernetes API and caches them in `resourceMaps`.
- If the settings are already cached, it returns the cached map.
- Otherwise, it fetches the settings list, copies each item, and stores them in a map keyed by setting name.
- It then caches the map in `resourceMaps` and returns it.
- Returns a non-nil map or an error; does not return nil map without error.

---

### Relevant snippet from `upgrade/v18xto190/upgrade.go` (lines 50-90):
```go
func upgradeSettings(namespace string, lhClient *lhclientset.Clientset, resourceMaps map[string]interface{}) (err error) {
	defer func() {
		err = errors.Wrapf(err, upgradeLogPrefix+"upgrade settings failed")
	}()

	settingsMap, err := upgradeutil.ListAndUpdateSettingsInProvidedCache(namespace, lhClient, resourceMaps)
	if err != nil {
		if apierrors.IsNotFound(err) {
			return nil
		}
		return errors.Wrapf(err, "failed to list all existing Longhorn settings during the settings upgrade")
	}

	if err := updateOrphanResourceAutoDeletionSetting(settingsMap); err != nil {
		return errors.Wrapf(err, "failed to update orphan resource auto-deletion settings")
	}

	return nil
}
```
- This function upgrades settings during the upgrade process.
- It calls `ListAndUpdateSettingsInProvidedCache` to get the settings map.
- If the settings are not found, it returns nil (no error).
- It then calls `updateOrphanResourceAutoDeletionSetting` to update the orphan resource auto-deletion setting.
- Returns any error encountered.

---

### Summary of relevant constants from `types/setting.go`:
```go
SettingNameOrphanAutoDeletion         = SettingName("orphan-auto-deletion")
SettingNameOrphanResourceAutoDeletion = SettingName("orphan-resource-auto-deletion")
```
- These constants represent the old and new setting names related to orphan resource auto-deletion.

---

### Summary of relevant constants from `types/setting.go` (OrphanResourceType values):
```go
OrphanResourceTypeReplicaData     = OrphanResourceType("replicaData")
OrphanResourceTypeEngineInstance  = OrphanResourceType("engineInstance")
OrphanResourceTypeReplicaInstance = OrphanResourceType("replicaInstance")
```
- These are the valid orphan resource types used as keys in the map returned by `UnmarshalOrphanResourceTypes`.

---

### Summary of relevant type from `k8s/pkg/apis/longhorn/v1beta2/setting.go` (lines 16-21):
```go
Setting struct {
	metav1.TypeMeta   `json:",inline"`
	metav1.ObjectMeta `json:"metadata,omitempty"`

	Value string `json:"value"`
}
```
- The `Setting` type has a `Value` field which is a string representing the setting's value.

---

These snippets provide the context for the unmarshaling of the orphan resource types setting, the upgrade logic that updates this setting, and the caching/loading of settings during the upgrade. They are directly relevant to the user comment about ensuring `UnmarshalOrphanResourceTypes` never returns a nil map without error and the handling of blank string values resulting in all resource types being off.

</details>



---

<details>
<summary>✏️ Learnings added</summary>

```
Learnt from: COLDTURNIP
PR: longhorn/longhorn-manager#3694
File: upgrade/v18xto190/upgrade.go:139-151
Timestamp: 2025-04-24T04:59:59.288Z
Learning: The UnmarshalOrphanResourceTypes function in Longhorn returns a non-nil map with all resource types set to false when given an empty string input. It returns nil only when there's an error (specifically when invalid resource types are found).
```

> Note: Learnings are effective only in the context of similar code segments. To apply general coding guidelines, please configure [review instructions](https://docs.coderabbit.ai/guides/review-instructions). You can manage existing learnings in the [app](https://app.coderabbit.ai/learnings).

</details>




<!-- <agentComponentType>chat-answer</agentComponentType> -->

<!-- <rawResChunk><answer><details>
<summary>🧩 Analysis chain</summary>

🏁 Script executed:
```
#!/bin/bash
# Get the implementation of UnmarshalOrphanResourceTypes
grep -r "func UnmarshalOrphanResourceTypes" --include="*.go" .
cat $(grep -l "func UnmarshalOrphanResourceTypes" --include="*.go" .)
```

Length of output: 281

---



</details>

@COLDTURNIP You're absolutely right. Looking at the implementation of `UnmarshalOrphanResourceTypes`, I can confirm it behaves exactly as you described:

```go
func UnmarshalOrphanResourceTypes(resourceTypesSetting string) (map[OrphanResourceType]bool, error) {
	resourceTypes := map[OrphanResourceType]bool{
		OrphanResourceTypeReplicaData:     false,
		OrphanResourceTypeEngineInstance:  false,
		OrphanResourceTypeReplicaInstance: false,
	}
	// ... processing logic ...
	if len(invalidItems) > 0 {
		return nil, fmt.Errorf("invalid orphan resource types: %s", strings.Join(invalidItems, ", "))
	}
	return resourceTypes, nil
}
```

The function immediately initializes a non-nil map with all types set to `false`, and only returns `nil` when there's an error with invalid resource types. For an empty string input, it returns the initialized map with all values `false`.

You're correct that no additional nil check is needed - the code as implemented will work properly.

---

<details>
<summary>🧬 Code Graph Analysis Results</summary>

### Relevant snippets from `types/setting.go` (lines 1802-1826):
```go
func UnmarshalOrphanResourceTypes(resourceTypesSetting string) (map[OrphanResourceType]bool, error) {
	resourceTypes := map[OrphanResourceType]bool{
		OrphanResourceTypeReplicaData:     false,
		OrphanResourceTypeEngineInstance:  false,
		OrphanResourceTypeReplicaInstance: false,
	}

	resourceTypesSetting = strings.Trim(resourceTypesSetting, " ")
	invalidItems := make([]string, 0, len(resourceTypesSetting))
	if resourceTypesSetting != "" {
		resourceTypeList := strings.Split(resourceTypesSetting, ";")
		for _, item := range resourceTypeList {
			resourceType := OrphanResourceType(strings.Trim(item, " "))
			if _, ok := resourceTypes[resourceType]; ok {
				resourceTypes[resourceType] = true
			} else {
				invalidItems = append(invalidItems, item)
			}
		}
	}
	if len(invalidItems) > 0 {
		return nil, fmt.Errorf("invalid orphan resource types: %s", strings.Join(invalidItems, ", "))
	}
	return resourceTypes, nil
}
```
- This function unmarshals a semicolon-separated string of orphan resource types into a map of `OrphanResourceType` keys to boolean values.
- It initializes the map with all known resource types set to `false`.
- If the input string is empty or blank, it returns the map with all values `false`.
- If invalid resource types are found, it returns an error and `nil` map.
- Otherwise, it returns the map with the specified resource types set to `true`.
- This function can return a `nil` map only if there is an error (invalid items).

---

### Relevant snippet from `upgrade/v18xto190/upgrade.go` (lines 110-138):
```go
func updateOrphanResourceAutoDeletionSetting(settingsMap map[string]*v1beta2.Setting) (err error) {
	oldSetting, exist := settingsMap[string(types.SettingNameOrphanAutoDeletion)]
	if !exist {
		return nil
	}
	newSetting, exist := settingsMap[string(types.SettingNameOrphanResourceAutoDeletion)]
	if !exist {
		newSetting = &v1beta2.Setting{
			ObjectMeta: metav1.ObjectMeta{
				Name: string(types.SettingNameOrphanResourceAutoDeletion),
			},
		}
		settingsMap[string(types.SettingNameOrphanResourceAutoDeletion)] = newSetting
	}
	resourceTypes, err := types.UnmarshalOrphanResourceTypes(newSetting.Value)
	if err != nil {
		return err
	}

	resourceTypes[types.OrphanResourceTypeReplicaData] = oldSetting.Value == "true"
	enabledResourceType := make([]string, 0, len(resourceTypes))
	for rt, enabled := range resourceTypes {
		if enabled {
			enabledResourceType = append(enabledResourceType, string(rt))
		}
	}
	newSetting.Value = strings.Join(enabledResourceType, ";")
	return nil
}
```
- This function updates the orphan resource auto-deletion setting during upgrade.
- It reads the old setting `orphan-auto-deletion` and the new setting `orphan-resource-auto-deletion`.
- If the new setting does not exist, it creates a new empty setting object and adds it to the map.
- It calls `UnmarshalOrphanResourceTypes` on the new setting's value.
- It sets the `OrphanResourceTypeReplicaData` key in the map to `true` if the old setting's value is `"true"`.
- It then rebuilds the new setting's value string by joining all enabled resource types with semicolons.
- Returns any error from unmarshaling or nil on success.

---

### Relevant snippet from `upgrade/util/util.go` (lines 392-413):
```go
func ListAndUpdateSettingsInProvidedCache(namespace string, lhClient *lhclientset.Clientset, resourceMaps map[string]interface{}) (map[string]*longhorn.Setting, error) {
	if v, ok := resourceMaps[types.LonghornKindSetting]; ok {
		return v.(map[string]*longhorn.Setting), nil
	}

	settings := map[string]*longhorn.Setting{}
	settingList, err := lhClient.LonghornV1beta2().Settings(namespace).List(context.TODO(), metav1.ListOptions{})
	if err != nil {
		return nil, err
	}
	for _, setting := range settingList.Items {
		settingCopy := longhorn.Setting{}
		if err := copier.Copy(&settingCopy, setting); err != nil {
			return nil, err
		}
		settings[setting.Name] = &settingCopy
	}

	resourceMaps[types.LonghornKindSetting] = settings

	return settings, nil
}
```
- This function lists Longhorn settings from the Kubernetes API and caches them in `resourceMaps`.
- If the settings are already cached, it returns the cached map.
- Otherwise, it fetches the settings list, copies each item, and stores them in a map keyed by setting name.
- It then caches the map in `resourceMaps` and returns it.
- Returns a non-nil map or an error; does not return nil map without error.

---

### Relevant snippet from `upgrade/v18xto190/upgrade.go` (lines 50-90):
```go
func upgradeSettings(namespace string, lhClient *lhclientset.Clientset, resourceMaps map[string]interface{}) (err error) {
	defer func() {
		err = errors.Wrapf(err, upgradeLogPrefix+"upgrade settings failed")
	}()

	settingsMap, err := upgradeutil.ListAndUpdateSettingsInProvidedCache(namespace, lhClient, resourceMaps)
	if err != nil {
		if apierrors.IsNotFound(err) {
			return nil
		}
		return errors.Wrapf(err, "failed to list all existing Longhorn settings during the settings upgrade")
	}

	if err := updateOrphanResourceAutoDeletionSetting(settingsMap); err != nil {
		return errors.Wrapf(err, "failed to update orphan resource auto-deletion settings")
	}

	return nil
}
```
- This function upgrades settings during the upgrade process.
- It calls `ListAndUpdateSettingsInProvidedCache` to get the settings map.
- If the settings are not found, it returns nil (no error).
- It then calls `updateOrphanResourceAutoDeletionSetting` to update the orphan resource auto-deletion setting.
- Returns any error encountered.

---

### Summary of relevant constants from `types/setting.go`:
```go
SettingNameOrphanAutoDeletion         = SettingName("orphan-auto-deletion")
SettingNameOrphanResourceAutoDeletion = SettingName("orphan-resource-auto-deletion")
```
- These constants represent the old and new setting names related to orphan resource auto-deletion.

---

### Summary of relevant constants from `types/setting.go` (OrphanResourceType values):
```go
OrphanResourceTypeReplicaData     = OrphanResourceType("replicaData")
OrphanResourceTypeEngineInstance  = OrphanResourceType("engineInstance")
OrphanResourceTypeReplicaInstance = OrphanResourceType("replicaInstance")
```
- These are the valid orphan resource types used as keys in the map returned by `UnmarshalOrphanResourceTypes`.

---

### Summary of relevant type from `k8s/pkg/apis/longhorn/v1beta2/setting.go` (lines 16-21):
```go
Setting struct {
	metav1.TypeMeta   `json:",inline"`
	metav1.ObjectMeta `json:"metadata,omitempty"`

	Value string `json:"value"`
}
```
- The `Setting` type has a `Value` field which is a string representing the setting's value.

---

These snippets provide the context for the unmarshaling of the orphan resource types setting, the upgrade logic that updates this setting, and the caching/loading of settings during the upgrade. They are directly relevant to the user comment about ensuring `UnmarshalOrphanResourceTypes` never returns a nil map without error and the handling of blank string values resulting in all resource types being off.

</details>



---

<details>
<summary>✏️ Learnings added</summary>

```
Learnt from: COLDTURNIP
PR: longhorn/longhorn-manager#3694
File: upgrade/v18xto190/upgrade.go:139-151
Timestamp: 2025-04-24T04:59:59.288Z
Learning: The UnmarshalOrphanResourceTypes function in Longhorn returns a non-nil map with all resource types set to false when given an empty string input. It returns nil only when there's an error (specifically when invalid resource types are found).
```

> Note: Learnings are effective only in the context of similar code segments. To apply general coding guidelines, please configure [review instructions](https://docs.coderabbit.ai/guides/review-instructions). You can manage existing learnings in the [app](https://app.coderabbit.ai/learnings).

</details>

</answer></rawResChunk> -->

<!-- This is an auto-generated reply by CodeRabbit -->
created_at: 2025-04-24T05:00:01Z
author_association: NONE
diff_hunk: @@ -612,6 +627,12 @@ func (imc *InstanceManagerController) handlePod(im *longhorn.InstanceManager) er
 		return err
 	}
 
+	if isPodDeletedOrNotRunning {
+		if err := imc.deleteOrphans(im, true); err != nil {
+			log.WithError(err).Warnf("Failed to delete orphan CRs for stopped instance manager pod %v", im.Name)
+		}
+	}
+
path: controller/instance_manager_controller.go
user: login: derekbit
body: Move it to `imc.syncOrphans()`
created_at: 2025-04-24T05:20:44Z
author_association: MEMBER
diff_hunk: @@ -726,6 +746,10 @@ func GetOrphanChecksumNameForOrphanedDataStore(nodeID, diskName, diskPath, diskU
 	return orphanPrefix + util.GetStringChecksumSHA256(strings.TrimSpace(fmt.Sprintf("%s-%s-%s-%s-%s", nodeID, diskName, diskPath, diskUUID, dataStore)))
 }
 
+func GetOrphanChecksumNameForOrphanedInstance(instanceName, nodeID, dataEngine string) string {
+	return orphanPrefix + util.GetStringChecksumSHA256(strings.TrimSpace(fmt.Sprintf("%s-%s-%s", instanceName, nodeID, dataEngine)))
+}
path: types/types.go
user: login: COLDTURNIP
body: CR object name should conform the DNS subdomain name (RFC 1123):

- contain no more than 253 characters
- contain only lowercase alphanumeric characters, '-' or '.'
- start with an alphanumeric character
- end with an alphanumeric character

The 63 chars limitation is for labels (RFC 1123 and RFC 1035).

> The name of the CRD object itself must be a valid [DNS subdomain name](https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#dns-subdomain-names) derived from the defined resource name and its API group; see [how to create a CRD](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#create-a-customresourcedefinition) for more details. Further, the name of an object whose kind/resource is defined by a CRD must also be a valid DNS subdomain name.
created_at: 2025-04-24T06:36:36Z
author_association: MEMBER
diff_hunk: @@ -726,6 +746,10 @@ func GetOrphanChecksumNameForOrphanedDataStore(nodeID, diskName, diskPath, diskU
 	return orphanPrefix + util.GetStringChecksumSHA256(strings.TrimSpace(fmt.Sprintf("%s-%s-%s-%s-%s", nodeID, diskName, diskPath, diskUUID, dataStore)))
 }
 
+func GetOrphanChecksumNameForOrphanedInstance(instanceName, nodeID, dataEngine string) string {
+	return orphanPrefix + util.GetStringChecksumSHA256(strings.TrimSpace(fmt.Sprintf("%s-%s-%s", instanceName, nodeID, dataEngine)))
+}
path: types/types.go
user: login: coderabbitai[bot]
body: @COLDTURNIP Thank you for the clarification about DNS subdomain name requirements. You're absolutely right that the current implementation meets the RFC 1123 requirements:

- The total length of "orphan-" + SHA256 hash is 71 characters, well under the 253-character limit
- It contains only valid characters (lowercase alphanumeric and hyphen)
- It starts and ends with alphanumeric characters

The 63-character limit applies to labels rather than the object names themselves. The current implementation is correct as-is and doesn't need truncation.

---

<details>
<summary>✏️ Learnings added</summary>

```
Learnt from: COLDTURNIP
PR: longhorn/longhorn-manager#3694
File: types/types.go:749-751
Timestamp: 2025-04-24T06:37:32.418Z
Learning: Kubernetes Custom Resource (CR) object names must conform to DNS subdomain name requirements (RFC 1123): maximum 253 characters, only lowercase alphanumeric characters plus '-' or '.', and must start and end with alphanumeric characters. The 63-character limit applies to labels (RFC 1123 and RFC 1035), not object names.
```

> Note: Learnings are effective only in the context of similar code segments. To apply general coding guidelines, please configure [review instructions](https://docs.coderabbit.ai/guides/review-instructions). You can manage existing learnings in the [app](https://app.coderabbit.ai/learnings).

</details>




<!-- <agentComponentType>chat-answer</agentComponentType> -->

<!-- <rawResChunk><answer>@COLDTURNIP Thank you for the clarification about DNS subdomain name requirements. You're absolutely right that the current implementation meets the RFC 1123 requirements:

- The total length of "orphan-" + SHA256 hash is 71 characters, well under the 253-character limit
- It contains only valid characters (lowercase alphanumeric and hyphen)
- It starts and ends with alphanumeric characters

The 63-character limit applies to labels rather than the object names themselves. The current implementation is correct as-is and doesn't need truncation.

---

<details>
<summary>✏️ Learnings added</summary>

```
Learnt from: COLDTURNIP
PR: longhorn/longhorn-manager#3694
File: types/types.go:749-751
Timestamp: 2025-04-24T06:37:32.418Z
Learning: Kubernetes Custom Resource (CR) object names must conform to DNS subdomain name requirements (RFC 1123): maximum 253 characters, only lowercase alphanumeric characters plus '-' or '.', and must start and end with alphanumeric characters. The 63-character limit applies to labels (RFC 1123 and RFC 1035), not object names.
```

> Note: Learnings are effective only in the context of similar code segments. To apply general coding guidelines, please configure [review instructions](https://docs.coderabbit.ai/guides/review-instructions). You can manage existing learnings in the [app](https://app.coderabbit.ai/learnings).

</details>

</answer></rawResChunk> -->

<!-- This is an auto-generated reply by CodeRabbit -->
created_at: 2025-04-24T06:37:33Z
author_association: NONE
diff_hunk: @@ -988,6 +1003,32 @@ func (imc *InstanceManagerController) deleteInstanceManagerPDB(im *longhorn.Inst
 	return nil
 }
 
+func (imc *InstanceManagerController) syncOrphans(im *longhorn.InstanceManager) error {
+	// Instances are live inside the instance manager pod.
+	// Remove the instance orphan CRs when the instance manager pod is deleted or not running.
+	var isInstanceManagerPodDeletedOrNotRunning = false
+	if im.Status.CurrentState != longhorn.InstanceManagerStateRunning {
+		isInstanceManagerPodDeletedOrNotRunning = true
+	} else {
+		switch pod, err := imc.ds.GetPodRO(im.Namespace, im.Name); {
+		case datastore.ErrorIsNotFound(err):
+			isInstanceManagerPodDeletedOrNotRunning = true
+		case err != nil:
+			return errors.Wrapf(err, "failed to check instance manager %v pod state while sync orphans", im.Name)
+		default:
+			isInstanceManagerPodDeletedOrNotRunning = pod == nil
+		}
+	}
path: controller/instance_manager_controller.go
user: login: COLDTURNIP
body: Partially extracted from `areDangerZoneSettingsSyncedToIMPod`, but not share as a method since `areDangerZoneSettingsSyncedToIMPod` needs the pod object for further checks.
created_at: 2025-04-24T07:33:28Z
author_association: MEMBER

PR comments:
user: login: coderabbitai[bot]
created_at: 2025-04-17T10:09:09Z
body: <!-- This is an auto-generated comment: summarize by coderabbit.ai -->
<!-- This is an auto-generated comment: skip review by coderabbit.ai -->

> [!IMPORTANT]
> ## Review skipped
> 
> Auto reviews are disabled on this repository.
> 
> Please check the settings in the CodeRabbit UI or the `.coderabbit.yaml` file in this repository. To trigger a single review, invoke the `@coderabbitai review` command.
> 
> You can disable this status message by setting the `reviews.review_status` to `false` in the CodeRabbit configuration file.

<!-- end of auto-generated comment: skip review by coderabbit.ai -->
<!-- walkthrough_start -->

## Walkthrough

This set of changes introduces a comprehensive overhaul of orphan resource management in the codebase. The primary enhancements include the expansion of orphan types to distinguish between replica data, engine instances, and replica instances, along with the addition of a `dataEngine` attribute to the orphan specification. Controllers and validation logic are updated to handle these new types, enabling more granular detection, creation, and cleanup of orphan resources. New settings allow fine-grained control over auto-deletion behavior per orphan resource type. The upgrade logic ensures seamless migration from the old settings to the new schema, and the CRD is updated accordingly.

## Changes

| Files / Areas | Change Summary |
|---------------|---------------|
| `controller/instance_manager_controller.go` | Introduced robust orphan detection, creation, and deletion logic for engine and replica instance orphans; added helper methods and event handling for orphan resources; integrated orphan management into controller reconciliation loops. |
| `controller/orphan_controller.go` | Refactored orphan cleanup to handle new orphan types; added type-specific cleanup and condition update logic; integrated instance manager event handling. |
| `controller/node_controller.go`, `controller/node_controller_test.go` | Updated orphan type usage and auto-deletion logic to support new resource types and settings; adjusted tests accordingly. |
| `controller/setting_controller.go` | Added synchronization logic for the new orphan resource auto-deletion setting; migrated deprecated setting. |
| `datastore/longhorn.go` | Added methods to retrieve new orphan auto-deletion settings and to list engine/replica instance orphans. |
| `k8s/pkg/apis/longhorn/v1beta2/orphan.go`, `k8s/pkg/client/applyconfiguration/longhorn/v1beta2/orphanspec.go` | Added new orphan types, condition, parameters, and the `DataEngine` field to the orphan spec and apply configuration. |
| `k8s/crds.yaml` | Updated the Orphan CRD to include the optional `dataEngine` property. |
| `k8s/pkg/apis/longhorn/v1beta2/instancemanager.go` | Added a new `InstanceStateTerminated` constant. |
| `types/setting.go` | Introduced a new setting for orphan resource auto-deletion with validation and migration logic; defined new resource type constants. |
| `types/types.go` | Added label helpers and naming functions for new orphan types; updated label logic for replica data orphans. |
| `upgrade/v18xto190/upgrade.go` | Added migration logic for the new orphan resource auto-deletion setting; refactored upgrade resource handling. |
| `webhook/resources/orphan/mutator.go`, `webhook/resources/orphan/validator.go` | Updated label assignment and validation logic to support new orphan types and parameters. |

## Assessment against linked issues

| Objective (Issue #) | Addressed | Explanation |
|---------------------|-----------|-------------|
| Automatically detect and clean up orphaned engine or replica processes and related runtime resources if the corresponding resource has been deleted (#6764) | ✅ |  |
| Leverage the orphan controller for orphaned runtime resource cleanup (#6764) | ✅ |  |
| Support granular handling and auto-deletion settings for different orphan resource types (#6764) | ✅ |  |
| Ensure CRD, validation, upgrade, and controller logic are updated for new orphan types and settings (#6764) | ✅ |  |

<!-- walkthrough_end -->
<!-- internal state start -->


<!-- DwQgtGAEAqAWCWBnSTIEMB26CuAXA9mAOYCmGJATmriQCaQDG+Ats2bgFyQAOFk+AIwBWJBrngA3EsgEBPRvlqU0AgfFwA6NPEgQAfACgjoCEYDEZyAAUASpETZWaCrKNwSPbABsvkCiQBHbGlcSHFcLzpIACIAMxJqLnwKblhMKIpsDHE2FAxEXEwGDwZIzGxuaMg5SGxESkgAYQB5ABkAEWgAVRsAOQBJK3RaWn9EeuQkBw8vfAwiWGSMAHpZ+cWKDDMANgB2bYAWDRhYD2ZtLEERMUkPfFiwhGRbFGQCFGZuSLZs9EYyjAVSBsBhpDBIZiQWLJfgpMEZLI5DzwfKFDDFZAAd3UCCwuFOkFacwWS3ssgKJGYxzgqHi1Gw/le6C88CI5Ho2Pxjw8RPWpIAohgwcUfqErBR8Nx8Ig0L4ABStflWACUkDWJM2q2JGy2AEYAAx7ACs1IJL20zDe+A+vHwUj80nwDOKwMwaFIouq8jIDgoKKIj2ofkR8FyKIKRWkgdCzg8GGt6oaaHG+AY8GoUU5sHQN3tmOSAGtZmhaMhYzwJdxKF55PAlNl4LF4FFMPR/Mw7XRjr1rSXaOp4HNZZBaKnHOxqIPLnwmNkSAAPUKY5MVu11qICEiyOb0fHIz7fCfiOYjkiFeBeMsYXcE/zxfzoqJTYLoa+EpUacyWRosUXIBxOC4bgEv2iAMHUiBTvweJmnYKKlNgShllCJCYpABZbjw+AorgiDHP04LiLKNYADRNG0nQ9AMQyilOUT4kGaCxDQfCICwHigeB4xTmRe5qvgRDwAwUIwkokTiPMsKpOkbYhmGqKRliTpePQm7Ap29CxBKkJ8eGaIuucGDug0cqhqqHaETC7y6Qpj6ukZpAznMuASj4lCmqgoKYKQTI4WQSi7tam64CxDqxJEYiMGCpCTFg9Qhf6iBkQ4RAxRJAZoKekFsqeaaQSe7xfGgLp8aUCSAtwDqIFK+TwGoLK4LWWCZUwnxzOw6mMg46gqJEIl8P4xXpWE1qzk2RAMpOJ4VLQGZ4UYABi+A+Pg2KSfiSBkZldQNEZbD0Eo/gFmooRSn59BOqEDFXQSs4SdgTqxTQRBUMeWByo0/SqraGLIGkmnaJEtBkSi/YMJO61pDdHgvP4QTwGMULwPOUYwiWQh1Lgf4jTwyZlngLAZruIR4ZA7SUCQx3qMybEOkEIQtg6EjNmhWksAoh0qCd2ifgYP6c6oPXwLxsAStgCyvjgBDnDQqn4KEYw1fUW0MAW8aYkDpA3h4/jM6hdPBAUr67n6qUU/QmW6yzq6/aaHgCPL/zOI2zY3kG1NVhQ0IUJaeQMO2E6+FbqFXgd+BRvGCskGAwdoa1zDqG8UOQGk9qyoNtDyJuZBMyzURZJE4xS4T4jCbHZaMtwaA7bQvM9lCDJ7k5rDsMgolIFx+X5JAmIUw6TAUAFeTcquzND7NhR2woGBSBQMpvWFEW4X8jXcEJw6xz3hbhateQSMtzOSUwSjDBjBR/mRn2QBIsp1lNGBbW+m9w4buG8/oxjgFA/n8A81cEMQMgyhZYKBbtkLgvB+DCFEOIKQMh5DH2UILTQ2hdBgEMCYKANJJhXmLoA8gr0ojx3YFwKgaEALnBcF6DmSCTpaB0B/T+pgDBoDXssDsYkNBEHwBwAw0R+EGAsJAAAgv0fBwCogUOcPIe4UVvLSGAiUaKHg+xJw8AAA3aNQNAgpBLkHUUjEgKkoTaRHuo5I0kMAaAAMpVgYAY6yBJ1HNDhJgWxogDEFEyJFTEpwsCzlnsNTK6j1Q6g0C4yxBirgwJxpgERVh+iQGca4jABixhOgoMUDyyA2CYDUfExJ/heDSCPFBWRcSLFgkgOrP2XhEJRj4po7RuiUQkAMdQFydU8AkDIr4oS2ZlzIGKczR6NZ+AJxCl2SA9d5anD4LMQSDAyKUAlHwMEtAWTzDIjCO6rkoSzDQqgJi8QxBdiMF+ERXgWL3ytCPJQpRnA3N/pABcUoKAgJhNwbAAgWTCXYAOBRBgezkCnl5eYdwMBjI4S7BpBI/KbGHLOLx2AbgnlkY0iJYI0mOmdMiGCGiCCYswDYHFmS2kN3RAvLMl1mQsX9K+b0843kfL4F8n5QkXkNkarlLwjy3rzX4dEIwEAwBGF2ctSIFBlh6UjAAfUMsZCgsrxVuQoFwnhfCBFCNEeIwh9ApFUNkWCmKiiRyNgeDhCUtAUVRlasUvxkF7SVLiek3F9ljKeizCiMxBEIyPgALJukcj+bIrlJXtLfIndSlk1UiOqfrVeKiWQrnUTKx84pUzSEQIG7gBijkjCiN7V03A16STTS6PaDTrTqN9fpEgGbfpROgWIUm7hp4uQlbtXe/hBIUjnpLEgUhfjrMlf1KSVTXVkqSpy+m2B6XlrOEGygMhkxRDRSk6pigPD9HaK2pxSASXVTmJBH5JAloUGsWedKBi2D4kUEyBcNBrz0WtPBOpJ8+LOuagTMAYkzxQXiulXmBER7nG9YgWQ6J+KSmeTZP1Bkl0UC2owYiON1EQfRES/IeaywFsCmEU2jlx1xLKuUbgU91EjvrYoG9Z5FgHSMWeVGG7Gg2CxH4keC73VEalPQVABRJRVgunwP9ssKMYYYFhxAtG718fxe2sNDRRPMcsU0NjKAHhwbrdxhoqAWIJyMsNGEAhV0XW/QA0TAGr2JV5sI+NaF1HKakzJ+jp5xIqaqax5ARayB6JUU/EgXx15hFkFWSYmnTjyHLEoSkURRO9RIMccm4kVAXnULWZASh9OtNUvIVNiAsPJbPAl9RvSECgmnpBQ6+MLOMYXuFd006uMLiQE+4oj8OSnCbpx2yLpWNMjAqca1QNqjFQLNBEe8YlDHAABJGM9sCOjihkCglEONxsnK/OwgdEF8GeR4NRnLINug3g13yfAhQB8oQpv23kPEXAoJ6V7gRmp0O19KCNlkPS1aBDEAIEqq2ewhQaC8zbTW3rJBA0OUoP6uY6hkg4c5cKF9WEfD7e0+cbgyB/C4AZOCSSDsuRacjDpvggPid2Uxx1nG9Qq6ELGRJ9HiljbEfyFPL9ZIoPer4hZeH8zWvIBawUb7KS3v+wSDQZA5A0Ic682O3zrSWdFN+ZlBdkx6ziCbC2NuKSNx5aQC08gWG6ARvoPlklu20DG9oOou29RFuyZW6cVWY6+JC7a3cB4A9FY7npXLwHP3l3/ZxllygBm7gbojLj+aBg2284IHwN99S/rzYaLELIqLu7vHBs9ZI8AABeHgfpZqjDha0CvyBK8Cyr4t07yejUGjQVnr2e44nQCWysfoMz8ExAQsKFNHxvbp2gW9y72cbvF/fdA+86yZYqCryX1DVuq3pe7kXqmvO82aLPYiIslHyNwxbPwghMaLbBRCEPTGxBkUnzQanynm9cf9ymNMRNW9ci4wqxyTW5wvQhgGDaZABZDlMvEeFVSVAAcmx1EDmDTGTQXkB3jzz0klmElDrwC2qhgXpR2goDADGlZEmlPTwUsxPEAxs3OUEUsGESuWASnFuT4nuV5VejoOeVeWSBZU8HZT+S5WbEQGFRETwxCyrGZHTEQC4HUUTSZ3TQlF+hzWLQAG0vF/QABdUJJYDQWtSMBtLNPNLAdRcAygaVCHeVRDZVZyRTNVbhdRAwKAYRQQ29ejcQ9PKDUyZgYSAAKk0IDUQxDQ7VVVVAkyk1cMgHcLUM2A0Ihyh0VVVBWQR2Hn0PMM7SlQXRMOhyVQMMsPwGsNsPsKW1oCcIz0gFcI8K8OKCiODUSP8LcyYyCNDBCLCKsVKMh0QxBkQCaPKMoGgDDxRH/2qHwGWhiMuziO9QSNDSSKMIO1SMVTMLGNVXVWyIEKHgcMUAKJcNDBKMiJ8MqMlVVCQEKzqwSzlA51CO1HUKwy2gJiKybwdmWhBk2MqgXW0PGBzVaKaOsVW2GzoGaAwB7BPmcIYDlBuzuIOyBz9HmFVDlBuK8GWSGIoGVAhKhJhNWWVF0KSQyImLrSmMchmL8MlXmJsMWKiGWPyKSX+KKPWJCPaK2NmJ2NeENxIHeKG1O1oG+MaAZCu1+JIEBK3VBP9GBO0yUPBKKMRJeVhJRPiPRJSK/0oBxIsPxJyKWLyNWOEmKMpM2LSN8IsN2MQAtxV0ZJOyBlZPZPYE5O5JPkFKIH5JJwtIRP6OhNFORNRNGNxMMKlNMIyPlMJPoGJOVPJLcLVIOw6IoE1KSO1LeI+OZKNMuxNK3TNI8AtOSjsXqNOPCLeLsWSmBzqGTL5FTIh2sUzMQFVChKdMlOMOlPSO2PcisIJLsMVNk19NVM8PVMVRDKqLIFnRICpLSKwyOOEDyBYliGKhIAAG8ABfcUkY0syY8s2UpIz02sokpU0kwouUSEJswMxDWHGNb6CVKk7gOUCEuok4nMxo5sxyK07wh4iHJ47NVhMicgOgfMmDKEicvQqczEmcj06shUxc+s5clwtcrsxVLcvnAIyDBgJoxAYI48jUU8jctIi8so1hKQ4oG8nNV8tEys5Iss90rC+c3Iv89RMk1cgMutIMkChPMCzDUXaChoiI+CxVRC5oq8g7NC1hDC50iwjEuVT8vC78r0h3Rw/8lUwCs8mHOHSiuk+YVpa3UyCHXoEfeMzpIUyEu0pE5IDi98ni3CmkqsrImsgioSoilc0ShixyCijS14XU9eWShdBS3IG04UtSh0jSksrC7ix8LEmUr8/Sn870pc4ygC0iyMciiSyypAJo63b400zAjwuitM0QDM+kZAGCsJN4gsosu0ty3S7C6cnSl0zIhYhc/ywi4i0ysizcsKuExgRvEgLDc9SCuUd3KTBQi05QkUo8+KsSpDFC5i3qti7gbZFJaAULGYFMqxLDEaqsIayxBaC8UKf4uURy1S24lyuEzS9yt0tI2cuY/i4qwSlY4Soo8qkKyq7cmqiXOqlJQExS3krZD4bM2C+iiqhC3q+ypSsEy01nKasak88JYa0asiCeHRaS8gH6/iP6rRQoekn6iE1Ks4lJdSuE7Kgqjy4oLyisnK/Cusoysq4K7wtIiy6qnPEgbhP0QvG86QOSjcliutAaw868mQrNF4/qpm549ilGrira6Yny6wwVYVZhdEm7HavE7hXhfm7VMRUgAhN/A1GRL3ZRPg2PW6RWh0StEeMgySQHfwXlEBJFNEZeNmSEZJSxYRS4urKcBxatLDQ9DJYoM2ggK4y24eRpTk1s8NCjA9aQJWOqSIc9S9BKeYFze9GpZfAsbzGEdRAO9Kd6m20le2828SZ2vSBIC6B4KO6zeYWOlJB2/AJ2uYW3SAEDHnPIpJUgXAXoVCYRa8f1KYf0a3ay8GKGtAJ8sYUrEeDnTHJkJsGg/wfDJPE+OYMZL9NuTTUapJSa0axutAZugxOUB8RS/DE2sEH66e2ekxdmZezAVe6vdedRZUCjJzUXYO+gGpHHP0QdWFFRH9EguKTOjKZCLu2RSdEqUa25KE8qZAZmYJAAcTPGjvrpSVttxVzvzowAPPbsBx2nyRvjqQ8CLS3owGAbJR3st3XveFDx9kVw23/kIFvofSMlPWBh215TTEhiLytkensH9D6g/riU1qICnliL4DYHGGMjewYeqUUorg8Bmjf2KgHn7HmBrHwjfNqqw3bs/Q3UkM8mUU0lMUQdQZVytonoBqrDXu0ULvcHt2NUO1oDPmhmbxfo8BHSeyCjOGSA8D/0BCYOoWMaEJ1moDmUDGamoeEftjtPKn2XdF5guWoOuX5RxgYNECYKeVkTYPeTXVZW+V+U5XCF4P4Prl0duQiY4LZVif+W5UYL5RYN7kZHOCUAAG5mRfBoUmwc8WDyx4UjJfB3g19JJiTYokk3asLbdNUhUbDBb3LhaMjZVJdNAxb2nKCREpagE9V7BHBKF5a5FwUla21+mkZFwGQPAb4u8iDNESBBzvBcBJqQhOTdQDFBl1J+wtd8NdGzEfqDEtdjFvUo1EH3F7EN7ja6LJ71Hd7wYVGQlxr/rLElH15Z7skmRMpIJ3GwBNl99wV0BYgTlho+JJCHluJymp90UCQOd7Ga4XaCQbsFMkiwgSZeZtVe6nlHEOIQmcnj1WCmV2ConOCMmeDAVgVEsTgIWfIh75AyneCesBxb588p9YGXxyk8WjZVn0wiDAdRwI5HZZRQpMBGVmUaX0mOVMn5BsnmDj1Px+bOnRUDB0Sv0Ra9LxatUqDRmZaQE5bnlknTUsN3au00JVEB0h1QgqM+Ai0gKiMURvY2A+A6mMAOzWdoDdaUdP9EMZmYpllFx/InsCR6mAxnWI6+AsMWdOS8hPWx841pcDqzd2zghggGrurEdLVFAbV8Ns2SABXRdqEWRhdyHIQTN6gzMR43XKAoDN0T4d1QdUWN1/BZw4D0wF5gDhIi18G0gZASAc47whpLGSS+JhlBwsz9CAQKhrd17iSmRlchzcs0TF3uAG7462kyt+kHRcdNhkI6GsBQZ15hpfE6MGhSMKomQ7V3NmRiQqt/M1rAXV2akqNMtzUB9QgOdE1BcmVfl1AawuAK88UDsyJldgsF1qcYO9tgagbGMiBej3hYr0wWRC96A72gR/jAmfNQb/M2x3nVcIdp125u2IgVWLbJIEPMpgagdLHSZ+gYxLxrREACx4AscRxtF/hyogQNspHVMJWpdHZNx1QgnbpjTfgPSjBK60JTgvAFtGn0BGQ+xC0YQuMOdcPuBxDdPrd6Smj26F2BOd3ddaBp7jPqdgW0g+6t2zPrdjPM3owU5WxC5qiF5v6esQTyz/hmxsgp4uMPXy9FwqBM8H32AqAsOnw9DH1wudmLOmiCJoQtHbpt2bYs1alk8oo1t50ItIPtN+MIyRsTMXcSXW2ekoFzx8cMpgwMBaverSd/P2BkOk7yHerqd8CsGj4MvH3/05hbMRgBwhxSm8jkB1Fy6bAshaum3gyWR2Bz0JGyJHM6tOyIdIG3xRimwfYsMmjGht2DEyBwYsdvBu9g20iWvfhz7mxYGmLPOeIWddPp4duZYoIB2qo7FYEjFZAO2ShfcF4+Gm8PvUBH1/IUcqMmv3djv/vrwRvu5COtttbSOmuR7r50w/gM3V31Egf1uDt8yMwQ1+w3p26+kKt6YL78kLvFUrvQgi0uMo9iP7Azx8lvdvbfcj4Aepwp5Z2qGM8ufRvahuAJ5PdePCh+O3QiDZxieoJUBQ6BeamJsAPRreZa74wyd9HMZsYB79xbQpB6AFk9EAwsxQSUVj3C1mwVJ2Mc5OPuPS0AxnvrUPBs8sLgQIRqBQRqdmGe4qAS16UTeOxGQkJ/ZuPvvFs2GYot8d8fA99Q3S8xjrUDJLHIBrGzuyc3xE0wAMPym3PrxNkAxZEdOMvAdpf4eheRef9npCEmu/PL7sgnprRSosKW26/To2aQWvrAdQw9edYT8ChyAi4S+mD0th5nv8wKACwd5MQY8/GiXAmKvVWwmHhUmFWYmlX6Wla/KXPAqVTUwQjrWsKYjfWc2z1kg5veyhB+zKBBzigxyOKD+crObxi9XeaCTYcTnXYXPspDNlmGy9/3CH+BVVUN21gIXguSX6d6ndSIAQkVka1e/ikhtYUA56ujHDn11QyA4B2yoAkrqQ3bb9iK//QAVqQl4VRl22iI4hunhrhEsMMAy7HAMOZt4d+RRAgQgMP7ECl2FnZBsUHIGqZKBE1FJIeUQAHczOP4A8Exj6KrVYBzDeAZYkQFFVDKy2MdEXzM68JIAugI6kcQ8KEDQybA8zpYjoBGcIc3AqpLwJ+ZggBBQgsjCIK+BiCRSkgsUtYVUFQBGBGg/fiwJyqqgDOHA0jk0SMFxITB1A8koIO3ZWCn2tg2gVIIcFqDnBzAmQawPi5DQnOEOFLvgF8FYB/B/AoopWiYpBkoBSJOgQYEcHqCYhYIRAR4O3aJCDsqQx6mEnOKdcPgkAhMh8HBpdUzKnRQGr1VYwPNqh6hBKgwHMHBCWA1g64s5TsGOkDKQ8ankRlKABcnWqeFhuNxUFRD8BmgtwUAJT5nhpuDXf0HNwO4zClu11Y4i80Rr1Dbqy1dwhB1YTwBnqp1DUgt2yBI0USBQpYSuWKGYBShnnPHnWkbIXC141wgmi2TuG4AmKDQ5Sl9QXQABpUGBDSepzcfqyyIjs0O+bN0Yao1GgXwAiFPCnByw1wbEPcEvcEYzAPbhDgsEVRvhRHS4X8KQq3CZhwI04aCJBjMBERf1WEe0Ig5MinqyIhEaiMCEkiKgIQmwSMPCH2C3+igGFCVVczRCVhuItYbjyJ7w8oKhw75gEOYYqNIeJze8B1GV5hYnuqGDNqX0B7C9u8jTcYb+QlHYiABqwogbj3SqE8FeVQ9IZYiYrsi0qEOWGkUVGGuURRH/U0feklE4iShrA3Hs3RJEJY5Rb0e0UcMsRoi6BOMfUVOGIjyBce0EMZEWno5i9MoI9IZlqzFTuUGG+rTIoayFSS1dUstSZtIgtaK0jAdmDNpwwkyiw4cvLBeODHtwIMAGWdRSnHTYggNE6A3VJD3BXAad8MjSCTMCi0TgoKAAALXahtiiAiAc9B0VoBWD2o9fE+ncX3hccta9mbfoESAZ7tQGFtAulPCx6l10M4FTsXbRID7ik6BdI9hfTgQd0u2e7YggeLvqB0H6fwKnC8jC5DR8kZ7a+LKBfBFoMUu4rsSgynqkd16GEWQDfjmCBJJgV0XsFAKKLRAXIwQKoDCDiCyh6g0QZUHviwD3ZVsyEJQMUhJr6p76SSGcdnVNo9iSe+EArqeBImy176LbflhxF/b9ojaYBWCZQDNb0iUAoQXHkPmXQC47kgWbtkxLfHv5swfEDNqxKnirscceOK8N6FhKcomAiIc2COAZBRsSk/AT2DcmOC1l4eCY2PhnTfFUSwQnAy8TROdpHNBxUnEoHUj7QptRonaCLl3W5xOImAbkMQDOOkwud3gOvficPDqDGQHcfoBgDHkJYBMWCC/Mlmq27jhMqWkTYTLS3X7xNAUUAf1KXUHENkwIIRGce8J3GWIrJV43sQeTWpP9VUywPMa/01YiojAwNATP4C1B/VBmEtY1iWLNZljDUCtA/G4HzBbjVOeTFRIIQq5NJCgrdclDIz0JNSE8JAVqU9SsJTwduRsYkitz/q4AZx547sY7RfHt0bul9fJJw0rRm4dpZKMqSTykl4Ie6+fB0CWDABstmehkUuNOiyCUI/sWE/iV/QAl4p3gmUJ+g8DRZPjAOOMM9ugRI7Hss8TwYtNtjiTe8cGGAeQBSEqiDkLwe6eMjATfDrSkkrQVrFJnPT0lq6lnbweRxsDNADpZ4JSX8B7osQogVbf9kDMfGgSMQE2YFnYjFGVdXOR2DmUix8DTNJC6iAwQdiiR8B1EVnDbvhH/YCAauZYNHGjwKhYseSdQelOojxkFApMAAIVkCcl26tM5dNyFrbyBE08HKmSexHj6z7ODMuGVgARn0SUQ3LXwAzPpRozLwxwTWbMkzZDIJQY8HWKnUemQoosDAX6DjGqaIpioqLZtMvGpR4BTwgWBQNwC+zzBfGwzfxrQQpZxSHkCU0el+PlapTFW3BDKZv0gDZTZMwwAKHlJCLN0ppqoTadtJAkXjLpU4CqauVYTyFAOpg4lHux+rtVBRjpeInNMsaLSwk/FUua5lynqCUq1c+aaqHVmJdLEc45IETOvASyDsOpZoDdQcqgiISX6OeZAHkLKEHRYIPIREIHnaJmpC0uitWXqnMICwAADkQDLB/YpYDQLIBHxeBCxqck1hIn1Q9TpmlrAwNWP1iSg3ow4C0quE9jcpHMzSIjocwHFjTG+TiWKp4iwJop6JiDAxGyQEzMArJ5MJsIRCggfQbA7QAIqtnOCAtbQkCmjmBD9Cbh8kkhWRIxwg6u44UEOZvCXziSpwVmv0ibtEAkC6hogospJHwoABMgi7sNaFmS3tVaI010B+kQUaIJGamdoDbJkTdZWMu6TFvAzAEpzop6crPAop5TktEpy/ZKWkzX6FyAUxc/apQp4l5Zga9JOeomi4CNCyAjgLgHwoEVkRRFOEhyUkmQWwYnESijcXJjMQj0NAdFQcJgsxgsBcFmzB2W9A1YCJsxBge+Y/O4AFgiAywS4Y/IaLLB+FwUNACIrRqUhEM6qT+cWOlo/yJmgEf+ZWOVostRppYTHvrH1qYBQgrEpJDaJoBdEeuRMFRo0m6XTTx6bir1vfEBY1jgcyITLPEvZDPt5gr7L8a1npSM9/wKKbMCuGiCZAth8waIN4oEwlo6AeymIMw2OXRAIw7yf0GcoOX29jlgOaIFkDVg/ZogEi/gN1g5YVMM5CYASEJBgmzF9kq0IamtVz4bJ6U5YYpPUF+CeTUA6o3RVQTn6xTDFi/QJklLznbYC5cTKxfwX2ptLfg4OfHlMt6UGY38AAXiLp5kplcoFCd0UMxHLxS40oZQ4nHojE0lywDJVkpyXDylg+S3UIUuKULpyy8xJJR0wampKH5bKzJdkrXi5LxqPKvlcsC/TlKhmlSsZqWNqUVj+pDSuPkByrgZ8CQ8UZ5CALZAF4omqmUGTUHXZkMAC+qmhoorUYkBp69AonKLDHZmpq2kUQDuIVeYOqIJmjFbt6uFl1pNuZ0+1avKDWGStxOnBXg41Ua4iZecwH6k0X5DzhWsiOeybKBfbrgll1bB3tGo7mLFjJvgXFaECgkTciRB2d6qbljVgg5u7SRkIW0T6ac+AO0YeBzmHyj5+0MIW9CWG0QUYsMDzTxKhMiig8I2z6DkG3kygZsQF8Y3wNczNyci/MUSMesIQmkg0/MlzA9hVlQCwM74ICf6cHMCz/tA5I8VicgE8XoS+APiwFnOt8hzgx1NOXmcbJAh8dmFkhenmwvCXTJQuyyySHOp1z8RFkanP2R2H17FNSoqtcsH2AHBpxI0CfG1EjD7yp9h+3Kb1DpwzBk0C8U+RAsVmBpwrLkMUr5aJKzlL9c51LfORYsxXiBMpTQY9AbWrXb1wJlubFEVGKCbt8Q/gWHn6lwherQ1vqwoCZwDVEdrOLORRoxpVzOd0U49UTVWDaZb9i19GjAGGKnCJqIcya1NUURtKfg5NtG9pWWpSRNFK1Im8tS9UVRz1NNJozSJb3nUwKtsC61pJcyYHcBQF9pUZTEH4UXq3NYi1UONP7V2JB13iUIN6iriqwwp6iApcVhEVtMb52rVleyqfmAipVXwWQPgQmgJSuVmocLYUGKUj07ESqjqSMy6mSI/5Gq2ZqanuZ2JhEJaGsCGnGiTQrpyKHxCuDB73qagUGzcRmzOhzgXWVmpJHZv0SwYpNvAzLUUo0B9aSAlzY4D+GjJHo4ewjaCS0rQgCA50KkBoNjwADqOIMbXAqehWpi2m67MPFHySrrHFhiYxBVz17rh6AnS5HlDPyTdsSAtwb1r2BWhoRV2XkB2cnJOCoBV20IF7cdNupVxJkmwdAEBx/UBh1EG2/ECfQo6zxMQfoWFgSBvUbZwYbkb0ts2459QkQUU+FQRoMVEbQmKK0xWis+QUbMmCTAknNSMQWw8M4hMbd0M2DDaRFo2mzfZtGqokGtoQcraIEq1JaatBBBKU6Vi2Srph7ARLTWBS11apw6WlYIzoVWi5cto8nKTTvUECB/RbiCrVVtkB87Ut98VUJDtgBja5QnShoozuZ3Q0uRVYVUBaMsQPMed1WuYLVoF3xEhdWSkXdkDF3JaHd/O++NLrlURa5dC8hXfpWi1GBAONU++nlqNYFaql4zc1kanqVALyE5EsyTHQ7ENzdpedfaQNuELgK7JghDNQsqzVu4U1OahieJJAR/jOGKe/0BZMwBNzDxzLMvaIAknDQ5eu8ShBhAtjQEHpT0rvsvGD5+gnNUEXHv3Th4k0m9ny89vhLQD7w+AKLOMPrAYZHjF95E4iKtH/CPqwVz0oSMtDmBZ9AsfKema1meTAyWZzvN+rEhvprdekbeHdbuAv1KB8F2uJJPR1norcIOwmwHOono7GcI1GbQWedOKAObRlEiNRPbjv3/i4GpMOzEpwWz4coI6iLoBgA+lpAvAgB8bW/W239ldtrGnGHTntyA4793ePiPUATjeS99tOQ/fqlBH9lEJgMqqBeIcbv1PGeSaDmbMa7wzVJG2FEBAdBngqxg7AZfUnskmoAdeoSviEQFmAmZfAnDBmW9kf0JK6CxwAAGq3wJ473H5YOyshYsV9kkvPUPHeBha1DGYfMp9RUY8H8AGEJJEgZQOyh0DP1fyTdrxyQHd1UEZhmWGDnJAhGkh37hQT0XZy/FyKlgqirI3orSdG/fgiGi43ly6A4hSiWnpKl7ibJJ4eI2wCdJh6l9/FVQ2sz6gTzq98wPBYobmDoH69WAGcYUYIU3iRiGRiPfxXBp5H7D49C0ukbfrh63xnpKIwbUPxNqFNVky5jxsSNn6/mTdP1b0e7mjVA1kYATenrAlvNLcznao60cyO+VIAC0DPAgWV2MCbDzgT6WgZmNAG3689cY2FhnFQCISmOeQo0arC9yJB9g+IjUfaPXzkloqsPR3PalR6dUMetVVMxK0motVyTbA0WxKiDS9ROm34Lygk7oQtwYhXGeNXpLgitwVatWeNWnoInZAaXeBusYQObSsMrQFQEYkXkUBl2CMGBMkHRNMgOWKOA1cJyqSSEITRiNDN6o0b8b0cqdZ5NJp9VMbqQIJ/WPAYpaQbNjOJlJHiYk6EnDOQmjbiJqFOWIRTBJ/YXoJJnzHJZkAD2VyFvwzB8TpTVhPG1ZxRBmFyPS3L1WgKxVvupEbLt4cmw8kd0sfDNswphBpioJ10inBWkUrU4+IrCShV3ibxaiNTkJuUBycmOPgkTTJ0mSLIPoFrnNZpydbyaxMnh1E0pkoc7nDqOB3q8p+ELQAWPdH8MXxpvC1CTMAQwAdbU1VUnVpFov0sXNebxDQAbiMoU+kElkK5nWmWcTCojkhJXAogvkmgL7d5ljP+JaNAW5COAs4l7h7cHZvANOk6S+w+kkuYLVVztTdJkI1iGbMIhEVGhtguXVWABGpzFImwKMeCddOiBfowALyxvQCea3aw1QmpsFW+D2j0oAk/yQjcFFCigR0oc6P7LqfoD6mAshptXPMtnFZryzB0Ek2IDzzSA8NacgI5nIJ3BGidoRknVwUo3k6oAnR9peXPlH3HWjbxrIosKcG8hYK8JxExpu3mYjYTJ5VE4RaWrEWKdvZmI6EokKYW368xHC0dQTOYBZTl4NMzJEDNcCbsTZiDiCM+rmQ25bVZoyRcYGsWMA7FsU14KVOVDeL7QNg5bgEv+ghL3ARQqCOUKiXCh4ls8Na3zMpnFKnF9kD4LsqunKuTZhxa2ccqiXqLlKADKyB/6MgR9GFsLMsCwvqJmLOl+eWCCktGW6A7QYCwnlkBxkLLSAAsF0C6A7ozjrVDS80aKIoDm8dJzU5AZfDhgs1fJjAFgJD0GAKgf+JQDyrvmLh8AuoAAJz6hlguVqgNNneNFjOpOZoreqvj2aqwcXQbgHlZ9Vn7/JGV/sdARv7zT8Mf6VDk3h5wXAhe7VgDRyn+lbjYDaeGizj0NE0AvMaXVADNacj5Blou6q+mNaqtKVAsOp1DHhZ1CMAYlkIYxsgCIWFknm18XUBoDvkaB5wOMfhRoBKsaB9QIMGKZJE7Bk5YZz9J8SErezPYnI02mqBaYZlgBWwYAJMRldJj8hVJrDGUDFB7h9w/Us0QeCavoAqzNxn0+zjueRiVYDatmLcZVZLC7XUZMIBhliHgVDwmIoUfeHUlyDE2kIDI6waKFi6TZ9Y819q35Kua9nr1NFuQyzkEnM83x3mUxDOx9mXaUMq2FnNn3Xj8zy+c0B8apnsa4Nf0L44W8NBqAJw/8COu4MYkr3kT/TqRq6tRL2nXjUkuxbIIhKx7o6Y4T4zhkbfvq16kGSRs272P3ryTWQCU3GHPC2s1j76gATAIfpcDWgx+OQq/Wz9TB5KCzxHhpjGO9jSQmQASyaReUAYIsw20/TGIGGQd8QQCBStVdycHGeoF3hi75Ik7RDBg26nNVjZQ7wLSkDvrWD77h8vEz6n9y3HzWReJR5IxgBnE827LJ4b9o8G+1e2p8A7G/I3h0lbjOG3B0IKJ03ShB3c1OIHvOmXjXbPDaNubYZLRze9FJ5s8sHDo9OZg28d0R9NgGHDw32G5pye4zbGoBhcb84Aljjv0X0EQI8UkjSv3I0IWyd1GtY/3eaibGySsotjJvO9pDlchaoWALsI6ihFYAbu3CGeA0BQP6+Z4MiAWG+QkAkHoQdwnA/iiIPAR8UNg2fpzQ5JhLGl+FDfxHLjl3RQo/uSMRvsFWirpV8qzfc9K/2IueRwB21Z2t+SQH1UMB40K8CQPAR9RWB/g4QeYOCHldslMQ5iufVlC5Docnf2ofoi7jdDrhyTYYcEAmHFV9R9VZWNsONjFco6rjy7tu23oM4pavfVvKVQLjbVdwqbpnHRjT5aj9q5o+KtlWdH7V/Etld7gCBFglh5YGdYD1gh2EeAagMkEj21Xo9qq7qY1b6mlb/jPJUxigWSt4wHLnodOxNhpNxIZpZiRoLVRPrsmv0/qMJwniwMJXOJwLW1RpjwLRqw6LsyOiGaY2xJ7AnICrIz09BMBZ4UbDjW6okiRQfTMJjk8ybQDTHfmExiUyLOpzDPQz4ayAPyAjkoYdGxEIiexI6jdX3g9VkeB6c7x9tfTBJ2oB33sC8yOU7apjBxLFudtVMsVHnpQyzId8+ozY+Bo0943NOjmlqvXNyHtyB8rGVAGxs4CWdgXG93vZJwGHbAXB/tbANg1DOvN7w1DwKjbPGEvOimgN6w01mcnOT+HiWSKt+4TtI0pSwjX9iI0CnajCqBa2rXx/44LCBO92j8r9PkuMMJ5InX8wrb/LifarTUX6VQyyAnjJAW2+Ty6izlasi8vZbne0NnCwDOX3gkPKDc5sSuMXG9vPLMqC/ZNNPlGPVtWovUZOvOVc69Qu/92Bsc8AwRB+rDRfedJ39oV1/QkmfqrJARnU0z5mHVtcUARnWjHkzLmkZv1/V9q7i+Si/1qv14n+xkDUgcAlpQj3qQnNmAFfd5Acwr7vCa5YI1ALDNZhbRSgi7Wu1szr3/WVqdcpIrAjyDtV1bNcrgKnpiKp6IJqdh0x0Mz5p1NcQBtODtUyz0NdBQw+BBcOIBoBm9VjOvXX22LtwWCzeSniJkbT6/Jh9PQu8csLng/C+95FpHl6sPEIxZzc2uUk56Xt91fBfhhqGCcWxktqXArgF6lrzif257d8aW680wuqx3ex+gtc+SCFTD2eSZRdzkiM8M8jOcsQA2b+d4HHd7WmoM2J71d2f0lPdWTXsKIMDOwLfnO9NliAzYpWDP6b822amMG+HLDxgMAYASkE5qahK2wQLbd9wbMxxSyaY1oUD0nHA9XPcPE3Lbadr4xtxK88+kpqtCiCnqiix2i3SQGUMHM+3Y2n6soci24Sj2k7rWrbNUlSKKA2IeoFH2UAx8wCqtc8zh7iQJuTwH3dDhUGZSRrPX2ok3rLeEj4f0+B0F9a2aU/5AD2fUQ99eZI59XkC1qjwDGxIbBZGOxnsdFnPSzgWEVhG4JsRrxcf3CXdLIufwQMf2W2Q9IJywtdiNFCQiXLxl5ZWjc0Ajj9MI2O4RLAJxuIg3ElAl6BH2ZmgfZbZUiHCRRzBiqyAxIABQCWiyCrukINBNfmZzv651eBvJTgztg6dIU3DHRn2r8Z3Md1eaMtNqxmi9/xC+8MwvJJaIZF5STcvd1llON3F5fghAQiyXuuml8CCvxtkKkbL5f1y+hhEsa3mBPeVQhrf6ueX7b2IEK9xFSvUr0aLqI5u5vLE+bqgIW88Rqf2C9KKdZp7Au2WIu/X83grdli+lrvYIW74pQ/cRilRGQlUZAFK+lv2YjYLPk246jocG3QODMJ6FDfMp6UzAdHdYPldhYevgXk8Ie5bBvhPvyzb7+F8YF/fMAa7s98D7+rKjYSc9JV4gDGQAfZqdrs9w64PrvfDHpPskuT4wB5tKhiomn6D7p/g+txTn7q++pBKDOyXnTQwAYEwSco06eCeqzh1/AkI/AaAchMVpqCIIqAyCehGggwRfxQEEyWVLPllSxw6AsqC5aEEYQK/aA2wERSQG2CHAyrJAVc7qAOAABmL37EDvl3yDg+ofULNF2AJASrJAL37QC98kBdguwI0HfOj8PA7fxv+OOoDN+lgLfg6POLQFlQ/x9AQAA= -->

<!-- internal state end -->
<!-- tips_start -->

---



<details>
<summary>🪧 Tips</summary>

### Chat

There are 3 ways to chat with [CodeRabbit](https://coderabbit.ai?utm_source=oss&utm_medium=github&utm_campaign=longhorn/longhorn-manager&utm_content=3694):

- Review comments: Directly reply to a review comment made by CodeRabbit. Example:
  - `I pushed a fix in commit <commit_id>, please review it.`
  - `Generate unit testing code for this file.`
  - `Open a follow-up GitHub issue for this discussion.`
- Files and specific lines of code (under the "Files changed" tab): Tag `@coderabbitai` in a new review comment at the desired location with your query. Examples:
  - `@coderabbitai generate unit testing code for this file.`
  -	`@coderabbitai modularize this function.`
- PR comments: Tag `@coderabbitai` in a new PR comment to ask questions about the PR branch. For the best results, please provide a very specific query, as very limited context is provided in this mode. Examples:
  - `@coderabbitai gather interesting stats about this repository and render them as a table. Additionally, render a pie chart showing the language distribution in the codebase.`
  - `@coderabbitai read src/utils.ts and generate unit testing code.`
  - `@coderabbitai read the files in the src/scheduler package and generate a class diagram using mermaid and a README in the markdown format.`
  - `@coderabbitai help me debug CodeRabbit configuration file.`

Note: Be mindful of the bot's finite context window. It's strongly recommended to break down tasks such as reading entire modules into smaller chunks. For a focused discussion, use review comments to chat about specific files and their changes, instead of using the PR comments.

### CodeRabbit Commands (Invoked using PR comments)

- `@coderabbitai pause` to pause the reviews on a PR.
- `@coderabbitai resume` to resume the paused reviews.
- `@coderabbitai review` to trigger an incremental review. This is useful when automatic reviews are disabled for the repository.
- `@coderabbitai full review` to do a full review from scratch and review all the files again.
- `@coderabbitai summary` to regenerate the summary of the PR.
- `@coderabbitai generate docstrings` to [generate docstrings](https://docs.coderabbit.ai/finishing-touches/docstrings) for this PR.
- `@coderabbitai generate sequence diagram` to generate a sequence diagram of the changes in this PR.
- `@coderabbitai resolve` resolve all the CodeRabbit review comments.
- `@coderabbitai configuration` to show the current CodeRabbit configuration for the repository.
- `@coderabbitai help` to get help.

### Other keywords and placeholders

- Add `@coderabbitai ignore` anywhere in the PR description to prevent this PR from being reviewed.
- Add `@coderabbitai summary` to generate the high-level summary at a specific location in the PR description.
- Add `@coderabbitai` anywhere in the PR title to generate the title automatically.

### Documentation and Community

- Visit our [Documentation](https://docs.coderabbit.ai) for detailed information on how to use CodeRabbit.
- Join our [Discord Community](http://discord.gg/coderabbit) to get help, request features, and share feedback.
- Follow us on [X/Twitter](https://twitter.com/coderabbitai) for updates and announcements.

</details>

<!-- tips_end -->
author_association: NONE
user: login: mergify[bot]
created_at: 2025-04-21T09:53:58Z
body: This pull request is now in conflict. Could you fix it @COLDTURNIP? 🙏
author_association: NONE
user: login: COLDTURNIP
created_at: 2025-04-23T06:59:32Z
body: After discussion, will move the orphan deletion from im monitor to im controller, to reflect the settings change.
author_association: MEMBER
user: login: derekbit
created_at: 2025-04-23T15:20:27Z
body: @COLDTURNIP CI failed.
author_association: MEMBER
user: login: derekbit
created_at: 2025-04-23T15:20:53Z
body: @coderabbitai review
author_association: MEMBER
user: login: coderabbitai[bot]
created_at: 2025-04-23T15:20:59Z
body: <!-- This is an auto-generated reply by CodeRabbit -->
<details>
<summary>:white_check_mark: Actions performed</summary>

Review triggered.

> Note: CodeRabbit is an incremental review system and does not re-review already reviewed commits. This command is applicable only when automatic reviews are paused.

</details>
author_association: NONE
user: login: derekbit
created_at: 2025-04-24T08:15:54Z
body: @COLDTURNIP CI still failed 
author_association: MEMBER