func (r *Resources) prepareDBServerPodTermination(ctx context.Context, log zerolog.Logger, p *v1.Pod, memberStatus api.MemberStatus, updateMember func(api.MemberStatus) error) error {
	if memberStatus.Phase.IsFailed() {
		log.Debug().Msg("Pod is already failed, safe to remove dbserver pod")
		return nil
	}
	if !memberStatus.Conditions.IsTrue(api.ConditionTypeMemberOfCluster) {
		log.Debug().Msg("Pod is not member of cluster")
		return nil
	}
	apiObject := r.context.GetAPIObject()
	if apiObject.GetDeletionTimestamp() != nil {
		log.Debug().Msg("Entire deployment is being deleted, safe to remove dbserver pod")
		return nil
	}
	dbserverDataWillBeGone := false
	if nodes, ok := r.context.GetCachedStatus().GetNodes(); ok {
		node, ok := nodes.Node(p.Spec.NodeName)
		if !ok {
			log.Warn().Msg("Node not found")
		} else if node.Spec.Unschedulable {
			if !r.context.GetSpec().IsNetworkAttachedVolumes() {
				dbserverDataWillBeGone = true
			}
		}
	}
	ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout())
	defer cancel()
	pvc, err := r.context.GetCachedStatus().PersistentVolumeClaimReadInterface().Get(ctxChild, memberStatus.PersistentVolumeClaimName, metav1.GetOptions{})
	if err != nil {
		log.Warn().Err(err).Msg("Failed to get PVC for member")
		return errors.WithStack(err)
	}
	if k8sutil.IsPersistentVolumeClaimMarkedForDeletion(pvc) {
		dbserverDataWillBeGone = true
	}
	if memberStatus.Phase == api.MemberPhaseDrain {
		dbserverDataWillBeGone = true
	}
	ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout())
	defer cancel()
	c, err := r.context.GetDatabaseClient(ctxChild)
	if err != nil {
		log.Debug().Err(err).Msg("Failed to create member client")
		return errors.WithStack(err)
	}
	ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout())
	defer cancel()
	cluster, err := c.Cluster(ctxChild)
	if err != nil {
		log.Debug().Err(err).Msg("Failed to access cluster")
		if r.context.GetSpec().Recovery.Get().GetAutoRecover() {
			if c, ok := k8sutil.GetContainerStatusByName(p, k8sutil.ServerContainerName); ok {
				if t := c.State.Terminated; t != nil {
					return nil
				}
			}
		}
		return errors.WithStack(err)
	}
	ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout())
	defer cancel()
	cleanedOut, err := cluster.IsCleanedOut(ctxChild, memberStatus.ID)
	if err != nil {
		return errors.WithStack(err)
	}
	if cleanedOut {
		if memberStatus.Conditions.Update(api.ConditionTypeCleanedOut, true, "CleanedOut", "") {
			if err := updateMember(memberStatus); err != nil {
				return errors.WithStack(err)
			}
		}
		log.Debug().Msg("DBServer is cleaned out.")
		return nil
	}
	if memberStatus.Conditions.IsTrue(api.ConditionTypeTerminated) {
		log.Warn().Msg("Member is already terminated before it could resign or be cleaned out. Not good, but removing dbserver pod because we cannot do anything further")
		if dbserverDataWillBeGone {
			memberStatus.Conditions.Update(api.ConditionTypeCleanedOut, true, "Draining server failed", "")
			memberStatus.CleanoutJobID = ""
			if memberStatus.Phase == api.MemberPhaseDrain {
				memberStatus.Phase = api.MemberPhaseCreated
			}
		} else if memberStatus.Phase == api.MemberPhaseResign {
			memberStatus.Phase = api.MemberPhaseCreated
		}
		if err := updateMember(memberStatus); err != nil {
			return errors.WithStack(err)
		}
		return nil
	}
	if memberStatus.Phase == api.MemberPhaseCreated {
		var jobID string
		ctxChild, cancelChild := context.WithTimeout(ctx, arangod.GetRequestTimeout())
		defer cancelChild()
		ctxJobID := driver.WithJobIDResponse(ctxChild, &jobID)
		if dbserverDataWillBeGone {
			log.Debug().Msg("Server is not yet cleaned out. Triggering a clean out now")
			if err := cluster.CleanOutServer(ctxJobID, memberStatus.ID); err != nil {
				log.Debug().Err(err).Msg("Failed to clean out server")
				return errors.WithStack(err)
			}
			memberStatus.Phase = api.MemberPhaseDrain
		} else {
			log.Debug().Msg("Temporary shutdown, resign leadership")
			if err := cluster.ResignServer(ctxJobID, memberStatus.ID); err != nil {
				log.Debug().Err(err).Msg("Failed to resign server")
				return errors.WithStack(err)
			}
			memberStatus.Phase = api.MemberPhaseResign
		}
		memberStatus.CleanoutJobID = jobID
		if err := updateMember(memberStatus); err != nil {
			return errors.WithStack(err)
		}
	} else if memberStatus.Phase == api.MemberPhaseDrain {
		ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout())
		defer cancel()
		agency, err := r.context.GetAgency(ctxChild)
		if err != nil {
			log.Debug().Err(err).Msg("Failed to create agency client")
			return errors.WithStack(err)
		}
		ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout())
		defer cancel()
		jobStatus, err := arangod.CleanoutServerJobStatus(ctxChild, memberStatus.CleanoutJobID, c, agency)
		if err != nil {
			log.Debug().Err(err).Msg("Failed to fetch job status")
			return errors.WithStack(err)
		}
		if jobStatus.IsFailed() {
			log.Warn().Str("reason", jobStatus.Reason()).Msg("Job failed")
			memberStatus.Phase = api.MemberPhaseCreated
			memberStatus.CleanoutJobID = ""
			if err := updateMember(memberStatus); err != nil {
				return errors.WithStack(err)
			}
			log.Error().Msg("Cleanout/Resign server job failed, continue anyway")
			return nil
		}
		if jobStatus.IsFinished() {
			memberStatus.CleanoutJobID = ""
			memberStatus.Phase = api.MemberPhaseCreated
		}
	} else if memberStatus.Phase == api.MemberPhaseResign {
		ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout())
		defer cancel()
		agency, err := r.context.GetAgency(ctxChild)
		if err != nil {
			log.Debug().Err(err).Msg("Failed to create agency client")
			return errors.WithStack(err)
		}
		ctxChild, cancel = context.WithTimeout(ctx, arangod.GetRequestTimeout())
		defer cancel()
		jobStatus, err := arangod.CleanoutServerJobStatus(ctxChild, memberStatus.CleanoutJobID, c, agency)
		if err != nil {
			log.Debug().Err(err).Msg("Failed to fetch job status")
			return errors.WithStack(err)
		}
		if jobStatus.IsFailed() {
			log.Warn().Str("reason", jobStatus.Reason()).Msg("Resign Job failed")
			memberStatus.Phase = api.MemberPhaseCreated
			memberStatus.CleanoutJobID = ""
			if err := updateMember(memberStatus); err != nil {
				return errors.WithStack(err)
			}
			log.Error().Msg("Cleanout/Resign server job failed, continue anyway")
			return nil
		}
		if jobStatus.IsFinished() {
			log.Debug().Str("reason", jobStatus.Reason()).Msg("Resign Job finished")
			memberStatus.CleanoutJobID = ""
			memberStatus.Phase = api.MemberPhaseCreated
			if err := updateMember(memberStatus); err != nil {
				return errors.WithStack(err)
			}
			return nil
		}
	}
	return errors.WithStack(errors.Newf("Server is not yet cleaned out"))
}
prepareDBServerPodTermination checks if the given dbserver pod is allowed to terminate
and if so, prepares it for termination.
It returns nil if the pod is allowed to terminate, an error otherwise.
package resources
import (
	"context"
	"time"
	"github.com/arangodb/kube-arangodb/pkg/util/errors"
	"github.com/rs/zerolog"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	driver "github.com/arangodb/go-driver"
	"github.com/arangodb/go-driver/agency"
	api "github.com/arangodb/kube-arangodb/pkg/apis/deployment/v1"
	"github.com/arangodb/kube-arangodb/pkg/util/arangod"
	"github.com/arangodb/kube-arangodb/pkg/util/k8sutil"
	v1 "k8s.io/api/core/v1"
)
func (r *Resources) prepareAgencyPodTermination(ctx context.Context, log zerolog.Logger, p *v1.Pod, memberStatus api.MemberStatus, updateMember func(api.MemberStatus) error) error {
	if memberStatus.Phase.IsFailed() {
		log.Debug().Msg("Pod is already failed, safe to remove agency serving finalizer")
		return nil
	}
	apiObject := r.context.GetAPIObject()
	if apiObject.GetDeletionTimestamp() != nil {
		log.Debug().Msg("Entire deployment is being deleted, safe to remove agency serving finalizer")
		return nil
	}
	agentDataWillBeGone := false
	if nodes, ok := r.context.GetCachedStatus().GetNodes(); ok {
		if !r.context.GetScope().IsNamespaced() && p.Spec.NodeName != "" {
			node, ok := nodes.Node(p.Spec.NodeName)
			if !ok {
				log.Warn().Msg("Node not found")
			} else if node.Spec.Unschedulable {
				agentDataWillBeGone = true
			}
		}
	}
	ctxChild, cancel := context.WithTimeout(ctx, k8sutil.GetRequestTimeout())
	defer cancel()
	pvc, err := r.context.GetCachedStatus().PersistentVolumeClaimReadInterface().Get(ctxChild, memberStatus.PersistentVolumeClaimName, metav1.GetOptions{})
	if err != nil {
		log.Warn().Err(err).Msg("Failed to get PVC for member")
		return errors.WithStack(err)
	}
	if k8sutil.IsPersistentVolumeClaimMarkedForDeletion(pvc) {
		agentDataWillBeGone = true
	}
	if !agentDataWillBeGone {
		log.Debug().Msg("Pod is just being restarted, safe to terminate agency pod")
		return nil
	}
	log.Debug().Msg("Agent data will be gone, so we will check agency serving status first")
	ctxChild, cancel = context.WithTimeout(ctx, time.Second*15)
	defer cancel()
	ctxLeader := agency.WithAllowNoLeader(ctxChild)  
	agencyConns, err := r.context.GetAgencyClients(ctxLeader, func(id string) bool { return id != memberStatus.ID })
	if err != nil {
		log.Debug().Err(err).Msg("Failed to create member client")
		return errors.WithStack(err)
	}
	if len(agencyConns) == 0 {
		log.Debug().Err(err).Msg("No more remaining agents, we cannot delete this one")
		return errors.WithStack(errors.Newf("No more remaining agents"))
	}
	if err := agency.AreAgentsHealthy(ctxLeader, agencyConns); err != nil {
		log.Debug().Err(err).Msg("Remaining agents are not healthy")
		return errors.WithStack(err)
	}
	if memberStatus.Conditions.Update(api.ConditionTypeAgentRecoveryNeeded, true, "Data Gone", "") {
		if err := updateMember(memberStatus); err != nil {
			return errors.WithStack(err)
		}
	}
	log.Debug().Msg("Agent is ready to be completely recovered.")
	return nil
}