func (ext *Checker) waitForPodStart(ctx context.Context) chan error {
	ext.log("waiting for pod to be running")
	outChan := make(chan error, 50)
	podClient := ext.KubeClient.CoreV1().Pods(ext.Namespace)
	go func() {
		ext.wg.Add(1)
		defer ext.wg.Done()
		for {
			ext.log("starting pod running watcher")
			pods, err := podClient.List(ctx, metav1.ListOptions{
				LabelSelector: kuberhealthyRunIDLabel + "=" + ext.currentCheckUUID,
			})
			if err != nil {
				outChan <- err
				return
			}
			if pods.Size() == 0 {
				outChan <- ErrPodDeletedBeforeRunning
				return
			}
			watcher, err := podClient.Watch(ctx, metav1.ListOptions{
				LabelSelector: kuberhealthyRunIDLabel + "=" + ext.currentCheckUUID,
			})
			if err != nil {
				outChan <- err
				return
			}
			for e := range watcher.ResultChan() {
				ext.log("got an event while waiting for pod to start running")
				if e.Type == watch.Deleted {
					ext.log("the khcheck check pod is deleted, waiting for start failed!")
					outChan <- ErrPodDeletedBeforeRunning
					watcher.Stop()
					return
				}
				p, ok := e.Object.(*apiv1.Pod)
				if !ok {
					ext.log("got a watch event for a non-pod object and ignored it")
					continue
				}
				for _, containerStat := range p.Status.ContainerStatuses {
					if containerStat.State.Waiting == nil {
						continue
					}
					if containerStat.State.Waiting.Reason == "ErrImagePull" {
						ext.log("pod had an error image pull")
						outChan <- errors.New(containerStat.State.Waiting.Reason)
						watcher.Stop()
						return
					}
				}
				ext.log("pod state is now:", string(p.Status.Phase))
				if p.Status.Phase == apiv1.PodRunning || p.Status.Phase == apiv1.PodFailed || p.Status.Phase == apiv1.PodSucceeded {
					ext.log("pod is now either running, failed, or succeeded")
					outChan <- nil
					watcher.Stop()
					return
				}
				select {
				case <-ext.shutdownCTX.Done():
					ext.log("external checker pod startup watch aborted due to check context being aborted")
					outChan <- nil
					watcher.Stop()
					return
				default:
				}
			}
		}
	}()
	return outChan
}
waitForPodStart returns a channel that notifies when the checker pod has advanced beyond 'Pending'
package external
import (
	"context"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"time"
	k8sErrors "k8s.io/apimachinery/pkg/api/errors"
	"github.com/google/uuid"
	log "github.com/sirupsen/logrus"
	"k8s.io/apimachinery/pkg/watch"
	"k8s.io/client-go/kubernetes"
	apiv1 "k8s.io/api/core/v1"
	policyv1 "k8s.io/api/policy/v1beta1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	typedv1 "k8s.io/client-go/kubernetes/typed/core/v1"
	khcheckv1 "github.com/kuberhealthy/kuberhealthy/v2/pkg/apis/khcheck/v1"
	khjobv1 "github.com/kuberhealthy/kuberhealthy/v2/pkg/apis/khjob/v1"
	khstatev1 "github.com/kuberhealthy/kuberhealthy/v2/pkg/apis/khstate/v1"
	"github.com/kuberhealthy/kuberhealthy/v2/pkg/checks/external/util"
)
const KHReportingURL = "KH_REPORTING_URL"
const KHRunUUID = "KH_RUN_UUID"
const KHDeadline = "KH_CHECK_RUN_DEADLINE"
const KHCheckNameAnnotationKey = "comcast.github.io/check-name"
const KHPodNamespace = "KH_POD_NAMESPACE"
const DefaultKuberhealthyReportingURL = "http://kuberhealthy.kuberhealthy.svc.cluster.local/externalCheckStatus"
const kuberhealthyRunIDLabel = "kuberhealthy-run-id"
const kuberhealthyCheckNameLabel = "kuberhealthy-check-name"
const defaultTimeout = time.Minute * 15
const defaultShutdownGracePeriod = time.Minute
var ErrPodRemovedExpectedly = errors.New("pod deleted expectedly")
var ErrPodRemovedUnexpectedly = errors.New("pod deleted unexpectedly")
var ErrPodDeletedBeforeRunning = errors.New("the khcheck check pod is deleted, waiting for start failed")
var DefaultName = "external-check"
var kubeConfigFile = filepath.Join(os.Getenv("HOME"), ".kube", "config")
var kuberhealthyNamespace = "kuberhealthy"
type Checker struct {
	CheckName                string  
	Namespace                string
	RunInterval              time.Duration  
	RunTimeout               time.Duration  
	KubeClient               *kubernetes.Clientset
	KHJobClient              *khjobv1.KHJobV1Client
	KHCheckClient            *khcheckv1.KHCheckV1Client
	KHStateClient            *khstatev1.KHStateV1Client
	PodSpec                  apiv1.PodSpec  
	OriginalPodSpec          apiv1.PodSpec  
	RunID                    string         
	KuberhealthyReportingURL string         
	ExtraAnnotations         map[string]string
	ExtraLabels              map[string]string
	Node                     string              
	currentCheckUUID         string              
	Debug                    bool                
	shutdownCTXFunc          context.CancelFunc  
	shutdownCTX              context.Context     
	wg                       sync.WaitGroup      
	hostname                 string              
	checkPodName             string              
	KHWorkload               khstatev1.KHWorkload
}
func init() {
	kuberhealthyNamespace = util.GetInstanceNamespace(kuberhealthyNamespace)
	log.Infoln("Kuberhealthy is located in the", kuberhealthyNamespace, "namespace.")
}
func New(client *kubernetes.Clientset, checkConfig *khcheckv1.KuberhealthyCheck, khCheckClient *khcheckv1.KHCheckV1Client, khStateClient *khstatev1.KHStateV1Client, reportingURL string) *Checker {
	return NewCheck(client, checkConfig, khCheckClient, khStateClient, reportingURL)
}
func NewCheck(client *kubernetes.Clientset, checkConfig *khcheckv1.KuberhealthyCheck, khCheckClient *khcheckv1.KHCheckV1Client, khStateClient *khstatev1.KHStateV1Client, reportingURL string) *Checker {
	if len(checkConfig.Namespace) == 0 {
		checkConfig.Namespace = "kuberhealthy"
	}
	log.Debugf("Creating external check from check config: %+v \n", checkConfig)
	return &Checker{
		Namespace:                checkConfig.Namespace,
		KHCheckClient:            khCheckClient,
		KHStateClient:            khStateClient,
		CheckName:                checkConfig.Name,
		KuberhealthyReportingURL: reportingURL,
		RunTimeout:               defaultTimeout,
		ExtraAnnotations:         make(map[string]string),
		ExtraLabels:              make(map[string]string),
		OriginalPodSpec:          checkConfig.Spec.PodSpec,
		PodSpec:                  checkConfig.Spec.PodSpec,
		KubeClient:               client,
		KHWorkload:               khstatev1.KHCheck,
	}
}
func NewJob(client *kubernetes.Clientset, jobConfig *khjobv1.KuberhealthyJob, khJobClient *khjobv1.KHJobV1Client, khStateClient *khstatev1.KHStateV1Client, reportingURL string) *Checker {
	if len(jobConfig.Namespace) == 0 {
		jobConfig.Namespace = "kuberhealthy"
	}
	log.Debugf("Creating kuberhealthy job from job config: %+v \n", jobConfig)
	return &Checker{
		Namespace:                jobConfig.Namespace,
		KHJobClient:              khJobClient,
		KHStateClient:            khStateClient,
		CheckName:                jobConfig.Name,
		KuberhealthyReportingURL: reportingURL,
		ExtraAnnotations:         make(map[string]string),
		ExtraLabels:              make(map[string]string),
		OriginalPodSpec:          jobConfig.Spec.PodSpec,
		PodSpec:                  jobConfig.Spec.PodSpec,
		KubeClient:               client,
		KHWorkload:               khstatev1.KHJob,
	}
}
func (ext *Checker) regeneratePodName() {
	var err error
	if len(ext.hostname) == 0 {
		ext.hostname, err = os.Hostname()
		if err != nil {
			log.Fatalln("Could not determine my hostname with error:", err)
		}
	}
	timeString := strconv.FormatInt(time.Now().Unix(), 10)
	ext.checkPodName = strings.ToLower(ext.CheckName + "-" + timeString)
}
func (ext *Checker) podName() string {
	return ext.checkPodName
}
func (ext *Checker) CurrentStatus() (bool, []string) {
	state, err := ext.getKHState()
	if err != nil {
		if k8sErrors.IsNotFound(err) || strings.Contains(err.Error(), "not found") {
			return true, []string{}
		}
		return false, []string{err.Error()}  
	}
	ext.log("length of error message slice:", len(state.Spec.Errors), state.Spec.Errors)
	if len(state.Spec.Errors) > 0 {
		ext.log("reporting check as OK=FALSE due to error messages > 0")
		return false, state.Spec.Errors
	}
	ext.log("reporting OK=TRUE due to error messages NOT > 0")
	return true, state.Spec.Errors
}
func (ext *Checker) Name() string {
	return ext.CheckName
}
func (ext *Checker) CheckNamespace() string {
	return ext.Namespace
}
func (ext *Checker) Interval() time.Duration {
	return ext.RunInterval
}
func (ext *Checker) Timeout() time.Duration {
	return ext.RunTimeout
}
func (ext *Checker) Run(ctx context.Context, client *kubernetes.Clientset) error {
	ext.KubeClient = client
	err := ext.setNewCheckUUID()
	if err != nil {
		return err
	}
	ext.log("Running external check iteration")
	err = ext.RunOnce(ctx)
	if err != nil && err.Error() == ErrPodRemovedExpectedly.Error() {
		ext.log("pod was removed during check expectedly. skipping this run")
		return ErrPodRemovedExpectedly
	}
	if err != nil {
		ext.log("Error with running external check:", err)
		return err
	}
	return nil
}
func (ext *Checker) getCheck() (*khcheckv1.KuberhealthyCheck, error) {
	log.Debugln("Fetching check", ext.CheckName, "in namespace", ext.Namespace)
	checkConfig, err := ext.KHCheckClient.KuberhealthyChecks(ext.Namespace).Get(ext.CheckName, metav1.GetOptions{})
	if err != nil {
		return &khcheckv1.KuberhealthyCheck{}, err
	}
	return &checkConfig, err
}
func (ext *Checker) cleanup(ctx context.Context) {
	ext.log("Evicting up any running pods with name", ext.podName())
	podClient := ext.KubeClient.CoreV1().Pods(ext.Namespace)
	checkLabelSelector := kuberhealthyCheckNameLabel + " = " + ext.CheckName
	ext.log("eviction: looking for pods with the label", checkLabelSelector)
	podList, err := podClient.List(ctx, metav1.ListOptions{
		LabelSelector: checkLabelSelector,
	})
	if err != nil {
		ext.log("error when searching for checker pods to clean up", err)
		return
	}
	wg := sync.WaitGroup{}
	for _, p := range podList.Items {
		ext.log("finding pods that are not in status.phase=Failed or status.phase=Succeeded")
		ext.log("pod:", p.Name, "is in status:", p.Status.Phase)
		if p.Status.Phase == apiv1.PodPending || p.Status.Phase == apiv1.PodUnknown || p.Status.Phase == apiv1.PodRunning {
			wg.Add(1)
			go func(p apiv1.Pod) {
				defer wg.Done()
				ext.log("evicting pod", p.GetName(), "from namespace", p.GetNamespace())
				err := ext.evictPod(ctx, p.GetName(), p.GetNamespace())
				if err != nil {
					ext.log("error killing pod", p.GetName()+":", err)
				}
			}(p)
		}
	}
	wg.Wait()
}
func (ext *Checker) evictPod(ctx context.Context, podName string, podNamespace string) error {
	podClient := ext.KubeClient.CoreV1().Pods(podNamespace)
	eviction := &policyv1.Eviction{
		ObjectMeta: metav1.ObjectMeta{
			Name:      podName,
			Namespace: podNamespace,
		},
	}
	err := podClient.Evict(ctx, eviction)
	if err != nil {
		ext.log("error when trying to cleanup/evict checker pod", podName, "in namespace", podNamespace+":", err)
		podExists, _ := util.PodNameExists(ext.KubeClient, podName, podNamespace)
		if podExists == true {
			err := util.PodKill(ext.KubeClient, podName, podNamespace, 30)
			if err != nil {
				ext.log("error killing pod", podName+":", err)
			}
		}
	}
	return err
}
func (ext *Checker) setUUID(uuid string) error {
	ext.log("Setting expected UUID to:", uuid)
	checkState, err := ext.getKHState()
	if err != nil && !(k8sErrors.IsNotFound(err) || strings.Contains(err.Error(), "not found")) {
		return fmt.Errorf("error setting uuid for check %s %w", ext.CheckName, err)
	}
	if err != nil && (k8sErrors.IsNotFound(err) || strings.Contains(err.Error(), "not found")) {
		ext.log("khstate did not exist, so a default object will be created")
		details := khstatev1.NewWorkloadDetails(ext.KHWorkload)
		details.Namespace = ext.CheckNamespace()
		details.AuthoritativePod = ext.hostname
		details.OK = true
		details.RunDuration = time.Duration(0).String()
		newState := khstatev1.NewKuberhealthyState(ext.CheckName, details)
		newState.Namespace = ext.Namespace
		ext.log("Creating khstate", newState.Name, newState.Namespace, "because it did not exist")
		_, err = ext.KHStateClient.KuberhealthyStates(ext.CheckNamespace()).Create(&newState)
		if err != nil {
			ext.log("failed to create a khstate after finding that it did not exist:", err)
			return err
		}
		checkState, err = ext.getKHState()
		if err != nil {
			ext.log("failed to fetch khstate khstate after creating it because it did not exist:", err)
			return err
		}
	}
	checkState.Spec.CurrentUUID = uuid
	ext.log("Updating khstate", checkState.Name, checkState.Namespace, "to setUUID:", checkState.Spec.CurrentUUID)
	_, err = ext.KHStateClient.KuberhealthyStates(ext.CheckNamespace()).Update(&checkState)
	tries := 0
	for {
		if tries >= 9 {
			return fmt.Errorf("failed to verify uuid match %s after %d tries", checkState.Spec.CurrentUUID, tries)
		}
		tries++
		ext.log("Waiting 1 second before checking " + ext.Name() + " uuid.")
		time.Sleep(time.Second * 1)
		extCheck, err := ext.KHStateClient.KuberhealthyStates(ext.CheckNamespace()).Get(ext.Name(), metav1.GetOptions{})
		if err != nil {
			ext.log("failed to get khstate while truing up check uuid:", err)
			continue
		}
		if checkState.Spec.CurrentUUID == extCheck.Spec.CurrentUUID {
			ext.log(ext.Name() + " verified uuid match.")
			break
		}
	}
	return err
}
func (ext *Checker) watchForCheckerPodDelete(ctx context.Context) chan error {
	ext.wg.Add(1)
	defer ext.wg.Done()
	listOptions := metav1.ListOptions{
		LabelSelector: kuberhealthyRunIDLabel + "=" + ext.currentCheckUUID,
	}
	waitForDeleteChan := make(chan error)
	watcher, err := ext.startPodWatcher(ctx, listOptions)
	if err != nil {
		waitForDeleteChan <- fmt.Errorf("error creating pod watcher: %w", err)
	}
	go func() {
		select {
		case <-ctx.Done():  
			ext.log("pod shutdown monitor stopping gracefully")
		case <-ext.waitForDeletedEvent(watcher):  
			ext.log("pod shutdown monitor witnessed the checker pod being removed")
			waitForDeleteChan <- fmt.Errorf("pod shutdown monitor witnessed the checker pod being removed")
		}
		watcher.Stop()
	}()
	return waitForDeleteChan
}
func (ext *Checker) startPodWatcher(ctx context.Context, listOptions metav1.ListOptions) (watch.Interface, error) {
	podClient := ext.KubeClient.CoreV1().Pods(ext.Namespace)
	ext.log("creating a pod watcher")
	return podClient.Watch(ctx, listOptions)
}
func (ext *Checker) waitForDeletedEvent(w watch.Interface) chan error {
	ext.wg.Add(1)
	defer ext.wg.Done()
	ext.log("starting pod shutdown watcher")
	var err error
	outChan := make(chan error, 2)
	go func() {
		for e := range w.ResultChan() {
			ext.log("got a result when watching for pod to remove")
			switch e.Type {
			case watch.Modified:  
				ext.log("checker pod shutdown monitor saw a modified event.")
				p, ok := e.Object.(*apiv1.Pod)
				if !ok {
					ext.log("checker pod shutdown monitor saw a modified event and the object was not a pod. skipped.")
					break
				}
				ext.log("checker pod shutdown monitor saw a modified event. the pod changed to ", p.Status.Phase)
			case watch.Deleted:  
				outChan <- nil
				return
			case watch.Error:
				ext.log("khcheck monitor saw an error event")
				o, ok := e.Object.(*metav1.Status)
				if !ok {
					err = fmt.Errorf("pod removal monitor had an error when watching for pod changes: " + o.String())
				} else {
					err = errors.New("unidentified error when watching for pod to be deleted")
				}
				outChan <- err
				return
			default:
				ext.log("pod removal monitor saw an irrelevant event type and ignored it:", e.Type)
			}
		}
		ext.log("wait for deleted event watcher has closed")
	}()
	ext.log("pod removal monitor ended unexpectedly")
	return outChan
}
func (ext *Checker) doFinalUpdateCheck(lastReportTime time.Time) (bool, error) {
	ext.log("witnessed checker pod removal. aborting watch for pod status report. check run skipped gracefully")
	currentUpdateTime, err := ext.getCheckLastUpdateTime()
	if err != nil {
		return false, err
	}
	ext.log("Last report time was:", lastReportTime, "vs", currentUpdateTime)
	if !currentUpdateTime.After(lastReportTime) {
		return false, nil
	}
	return true, nil
}
func (ext *Checker) newError(s string) error {
	return errors.New(ext.CheckNamespace() + "/" + ext.Name() + ": " + s)
}
func (ext *Checker) RunOnce(ctx context.Context) error {
	ext.shutdownCTX, ext.shutdownCTXFunc = context.WithCancel(ctx)
	defer ext.shutdownCTXFunc()
	defer ext.cleanup(ctx)
	ext.regeneratePodName()
	lastReportTime, err := ext.getCheckLastUpdateTime()
	if err != nil {
		return err
	}
	ext.log("Validating pod spec of external check")
	err = ext.validatePodSpec()
	if err != nil {
		return err
	}
	ext.log("Timeout set to", ext.RunTimeout.String())
	deadline := time.Now().Add(ext.RunTimeout)
	timeoutChan := time.After(ext.RunTimeout)
	ext.log("Configuring spec of external check")
	err = ext.configureUserPodSpec(deadline)
	if err != nil {
		return ext.newError("failed to configure pod spec for Kubernetes from user specified pod spec: " + err.Error())
	}
	ext.log("Running sanity check on check parameters")
	err = ext.sanityCheck()
	if err != nil {
		return err
	}
	ext.log("Waiting for all existing pods to clean up")
	select {
	case <-timeoutChan:
		ext.log("timed out waiting for all existing pods to clean up")
		errorMessage := "failed to see pod cleanup within timeout"
		return ext.newError(errorMessage)
	case err = <-ext.waitForAllPodsToClear(ctx):
		if err != nil {
			errorMessage := "error waiting for pod to clean up: " + err.Error()
			ext.log(err.Error())
			return ext.newError(errorMessage)
		}
	case <-ext.shutdownCTX.Done():
		ext.log("shutting down check. aborting wait for all pods to clean up")
		return nil
	}
	ext.log("No checker pods exist.")
	podShutdownWatchCtx, podShutdownWatchCtxCancel := context.WithCancel(ctx)
	podDeletedChan := ext.watchForCheckerPodDelete(podShutdownWatchCtx)
	defer podShutdownWatchCtxCancel()
	ext.log("creating pod for external check:", ext.CheckName)
	ext.log("checker pod annotations and labels:", ext.ExtraAnnotations, ext.ExtraLabels)
	createdPod, err := ext.createPod(ctx)
	if err != nil {
		ext.log("error creating pod")
		return ext.newError("failed to create pod for checker: " + err.Error())
	}
	ext.log("Check", ext.Name(), "created pod", createdPod.Name, "in namespace", createdPod.Namespace)
	select {
	case <-timeoutChan:  
		ext.log("timed out waiting for pod to startup")
		return ext.newError("failed to see pod running within timeout")
	case err := <-podDeletedChan:  
		if err != nil {
			ext.log("error from pod shutdown watcher when watching for checker pod to start:", err.Error())
			ext.log("pod removed unexpectedly while waiting for pod to start running")
			return ErrPodRemovedUnexpectedly
		}
		ext.log("pod removed expectedly. pod shutdown monitor shutting down")
		return ErrPodRemovedExpectedly
	case err = <-ext.waitForPodStart(ctx):  
		if err != nil {
			ext.cleanup(ctx)
			errorMessage := "error when waiting for pod to start: " + err.Error()
			ext.log(errorMessage)
			return ext.newError(errorMessage)
		}
		ext.log("External check pod is running:", ext.podName())
	case <-ext.shutdownCTX.Done():  
		ext.log("shutting down check. aborting watch for pod to start")
		return nil
	}
	ext.log("Waiting for pod status to be reported from pod", ext.podName(), "in namespace", ext.Namespace)
	select {
	case <-timeoutChan:  
		ext.log("timed out waiting for pod status to be reported")
		errorMessage := "timed out waiting for checker pod to report in"
		ext.log(errorMessage)
		return ext.newError(errorMessage)
	case err := <-podDeletedChan:  
		if err != nil {
			ext.log("error from pod shutdown watcher when watching for checker pod to report results:", err.Error())
			ext.log("pod removed unexpectedly while waiting for pod to report results")
			return ErrPodRemovedUnexpectedly
		}
		ext.log("pod removed expectedly. pod shutdown monitor shutting down")
		return ErrPodRemovedExpectedly
	case err = <-ext.waitForPodStatusUpdate(lastReportTime):  
		if err != nil {
			errorMessage := "found an error when waiting for pod status to update: " + err.Error()
			ext.log(errorMessage)
			return ext.newError(errorMessage)
		}
		ext.log("External check pod has reported status for this check iteration:", ext.podName())
	case <-ext.shutdownCTX.Done():  
		ext.log("shutting down check. aborting wait for pod status to update")
		return nil
	}
	podShutdownWatchCtxCancel()
	select {
	case <-timeoutChan:  
		errorMessage := "timed out waiting for pod to exit"
		ext.log(errorMessage)
		return ext.newError(errorMessage)
	case err = <-ext.waitForPodExit(ctx):  
		ext.log("External check pod is done running:", ext.podName())
		if err != nil {
			errorMessage := "found an error when waiting for pod to exit: " + err.Error()
			ext.log(errorMessage, err)
			return ext.newError(errorMessage)
		}
	case <-ext.shutdownCTX.Done():  
		ext.log("shutting down check. aborting wait for pod to be done running")
		return nil
	}
	ext.log("Run completed!")
	return nil
}
func (ext *Checker) log(s ...interface{}) {
	log.Infoln(ext.currentCheckUUID+" "+ext.Namespace+"/"+ext.CheckName+":", s)
}
func (ext *Checker) deletePod(podName string) error {
	ext.log("Deleting pod with name", podName)
	podClient := ext.KubeClient.CoreV1().Pods(ext.Namespace)
	gracePeriodSeconds := int64(1)
	deletionPolicy := metav1.DeletePropagationForeground
	err := podClient.Delete(context.TODO(), podName, metav1.DeleteOptions{
		GracePeriodSeconds: &gracePeriodSeconds,
		PropagationPolicy:  &deletionPolicy,
	})
	if err != nil && !(k8sErrors.IsNotFound(err) || strings.Contains(err.Error(), "not found")) {
		return err
	}
	return nil
}
func (ext *Checker) sanityCheck() error {
	if ext.Namespace == "" {
		return errors.New("check namespace can not be empty")
	}
	if ext.KubeClient == nil {
		return errors.New("kubeClient can not be nil")
	}
	if len(ext.PodSpec.Containers) == 0 {
		return errors.New("pod has no configured containers")
	}
	return nil
}
func (ext *Checker) getKHState() (khstatev1.KuberhealthyState, error) {
	return ext.KHStateClient.KuberhealthyStates(ext.Namespace).Get(ext.CheckName, metav1.GetOptions{})
}
func (ext *Checker) getCheckLastUpdateTime() (metav1.Time, error) {
	state, err := ext.getKHState()
	if err != nil && (k8sErrors.IsNotFound(err) || strings.Contains(err.Error(), "not found")) {
		return metav1.Time{}, nil
	}
	if state.Spec.LastRun.IsZero() {
		return metav1.Time{}, nil
	}
	return *state.Spec.LastRun, err
}
func (ext *Checker) waitForPodStatusUpdate(lastUpdateTime metav1.Time) chan error {
	ext.log("waiting for pod to report in to status page...")
	outChan := make(chan error, 50)
	ext.wg.Add(1)
	go func() {
		defer ext.wg.Done()
		for {
			time.Sleep(time.Second * 5)
			ext.log("waiting for external checker pod to report in...")
			select {
			case <-ext.shutdownCTX.Done():
				ext.log("aborting wait for external checker pod to report in due to context cancellation")
				outChan <- nil
				return
			default:
			}
			hasReported, err := ext.podHasReportedInAfterTime(lastUpdateTime)
			if err != nil {
				ext.log("Error checking if checker pod has reported in since last update time:", err)
				time.Sleep(time.Second)
				continue
			}
			if hasReported {
				ext.log("saw pod update since", lastUpdateTime)
				outChan <- nil
				return
			}
			ext.log("have not yet seen pod update since", lastUpdateTime)
		}
	}()
	return outChan
}
func (ext *Checker) podHasReportedInAfterTime(t metav1.Time) (bool, error) {
	currentUpdateTime, err := ext.getCheckLastUpdateTime()
	if err != nil {
		return false, err
	}
	ext.log("Last report time was:", t, "vs", currentUpdateTime)
	if currentUpdateTime.After(t.Time) {
		return true, nil
	}
	return false, nil
}
func (ext *Checker) waitForAllPodsToClear(ctx context.Context) chan error {
	ext.log("waiting for pod to clear")
	outChan := make(chan error, 2)
	podClient := ext.KubeClient.CoreV1().Pods(ext.Namespace)
	go func() {
		ext.wg.Add(1)
		defer ext.wg.Done()
		for {
			log.Debugln("Waiting for checker pod", ext.podName(), "to clear...")
			time.Sleep(time.Second * 5)
			select {
			case <-ext.shutdownCTX.Done():
				outChan <- nil
				return
			default:
			}
			p, err := podClient.Get(ctx, ext.podName(), metav1.GetOptions{})
			if err != nil {
				if k8sErrors.IsNotFound(err) || strings.Contains(err.Error(), "not found") {
					ext.log("all pods cleared")
					outChan <- nil
					return
				}
				outChan <- err
				return
			}
			ext.log("pod", ext.podName(), "still exists with status", p.Status.Phase, p.Status.Message, "- waiting for removal...")
		}
	}()
	return outChan
}
func (ext *Checker) waitForPodExit(ctx context.Context) chan error {
	ext.log("waiting for pod to exit")
	outChan := make(chan error, 50)
	podClient := ext.KubeClient.CoreV1().Pods(ext.Namespace)
	go func() {
		ext.wg.Add(1)
		defer ext.wg.Done()
		for {
			ext.log("starting pod exit watcher")
			pods, err := podClient.List(ctx, metav1.ListOptions{
				LabelSelector: kuberhealthyRunIDLabel + "=" + ext.currentCheckUUID,
			})
			if err != nil {
				outChan <- err
				return
			}
			var podExists bool
			for _, p := range pods.Items {
				if p.Status.Phase == apiv1.PodRunning || p.Status.Phase == apiv1.PodPending {
					podExists = true
					break
				}
			}
			if !podExists {
				outChan <- nil
				return
			}
			select {
			case <-ext.shutdownCTX.Done():
				ext.log("external checker pod aborted due to check context being aborted")
				outChan <- nil
				return
			default:
			}
			time.Sleep(time.Second * 5)  
		}
	}()
	return outChan
}