func (k *Kuberhealthy) reapKHStateResources(ctx context.Context) error {
	khStates, err := khStateClient.KuberhealthyStates(listenNamespace).List(metav1.ListOptions{})
	if err != nil {
		return fmt.Errorf("khState reaper: error listing khStates for reaping: %w", err)
	}
	khChecks, err := listUnstructuredKHChecks(ctx)
	if err != nil {
		return fmt.Errorf("khState reaper: error listing unstructured khChecks: %w", err)
	}
	khJobs, err := khJobClient.KuberhealthyJobs(listenNamespace).List(metav1.ListOptions{})
	if err != nil {
		return fmt.Errorf("khState reaper: error listing khJobs for reaping: %w", err)
	}
	log.Infoln("khState reaper: analyzing", len(khStates.Items), "khState resources")
	for _, khState := range khStates.Items {
		log.Debugln("khState reaper: analyzing khState", khState.GetName(), "in", khState.GetName())
		var foundKHCheck bool
		for _, kc := range khChecks.Items {
			khCheck, err := convertUnstructuredKhCheck(kc)
			if err != nil {
				log.Errorln("Error converting unstructured object to khcheck:", err)
				continue
			}
			log.Debugln("khState reaper:", khCheck.GetName(), "==", khState.GetName(), "&&", khCheck.GetNamespace(), "==", khState.GetNamespace())
			if khCheck.GetName() == khState.GetName() && khCheck.GetNamespace() == khState.GetNamespace() {
				log.Infoln("khState reaper:", khState.GetName(), "in", khState.GetNamespace(), "is still valid")
				foundKHCheck = true
				break
			}
		}
		var foundKHJob bool
		for _, kj := range khJobs.Items {
			log.Debugln("khState reaper:", kj.GetName(), "==", khState.GetName(), "&&", kj.GetNamespace(), "==", khState.GetNamespace())
			if kj.GetName() == khState.GetName() && kj.GetNamespace() == khState.GetNamespace() {
				log.Infoln("khState reaper:", khState.GetName(), "in", khState.GetNamespace(), "is still valid")
				foundKHJob = true
				break
			}
		}
		if !foundKHCheck && !foundKHJob {
			log.Infoln("khState reaper: removing khState", khState.GetName(), "in", khState.GetNamespace())
			err := khStateClient.KuberhealthyStates(khState.GetNamespace()).Delete(khState.GetName(), &metav1.DeleteOptions{})
			if err != nil {
				log.Errorln(fmt.Errorf("khState reaper: error when removing invalid khstate: %w", err))
			}
		}
	}
	return nil
}
reapKHStateResources runs a single audit on khState resources.  Any that don't have a matching khCheck are
deleted.
package main
import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"reflect"
	"strconv"
	"strings"
	"sync"
	"time"
	"github.com/google/uuid"
	log "github.com/sirupsen/logrus"
	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/watch"
	"k8s.io/client-go/kubernetes"
	khcheckv1 "github.com/kuberhealthy/kuberhealthy/v2/pkg/apis/khcheck/v1"
	khjobv1 "github.com/kuberhealthy/kuberhealthy/v2/pkg/apis/khjob/v1"
	khstatev1 "github.com/kuberhealthy/kuberhealthy/v2/pkg/apis/khstate/v1"
	"github.com/kuberhealthy/kuberhealthy/v2/pkg/checks/external"
	"github.com/kuberhealthy/kuberhealthy/v2/pkg/checks/external/status"
	"github.com/kuberhealthy/kuberhealthy/v2/pkg/health"
	"github.com/kuberhealthy/kuberhealthy/v2/pkg/masterCalculation"
	"github.com/kuberhealthy/kuberhealthy/v2/pkg/metrics"
)
type Kuberhealthy struct {
	Checks             []*external.Checker
	ListenAddr         string  
	MetricForwarder    metrics.Client
	overrideKubeClient *kubernetes.Clientset
	cancelChecksFunc   context.CancelFunc  
	cancelReaperFunc   context.CancelFunc  
	wg                 sync.WaitGroup      
	shutdownCtxFunc    context.CancelFunc  
	stateReflector     *StateReflector     
}
func NewKuberhealthy() *Kuberhealthy {
	kh := &Kuberhealthy{}
	kh.stateReflector = NewStateReflector()
	return kh
}
func (k *Kuberhealthy) setCheckExecutionError(checkName string, checkNamespace string, exErr error) error {
	details := khstatev1.NewWorkloadDetails(khstatev1.KHCheck)
	check, err := k.getCheck(checkName, checkNamespace)
	if err != nil {
		return err
	}
	if check.Namespace != "" {
		details.Namespace = check.CheckNamespace()
	}
	details.OK = false
	details.Errors = []string{"Check execution error: " + exErr.Error()}
	khc, err := k.getCheck(checkName, checkNamespace)
	if err != nil {
		return fmt.Errorf("Error when setting execution error on check %s %s %w", checkName, checkNamespace, err)
	}
	checkState, err := getCheckState(khc)
	if err != nil {
		return fmt.Errorf("Error when setting execution error on check (getting check state for current UUID) %s %s %w", checkName, checkNamespace, err)
	}
	details.CurrentUUID = checkState.CurrentUUID
	log.Debugln("Setting execution state of check", checkName, "to", details.OK, details.Errors, details.CurrentUUID, details.GetKHWorkload())
	err = k.storeCheckState(checkName, checkNamespace, details)
	if err != nil {
		return fmt.Errorf("Was unable to write an execution error to the CRD status with error: %w", err)
	}
	return nil
}
func (k *Kuberhealthy) setJobExecutionError(jobName string, jobNamespace string, exErr error) error {
	details := khstatev1.NewWorkloadDetails(khstatev1.KHJob)
	job, err := k.getJob(jobName, jobNamespace)
	if err != nil {
		return err
	}
	if job.Namespace != "" {
		details.Namespace = job.CheckNamespace()
	}
	details.OK = false
	details.Errors = []string{"Job execution error: " + exErr.Error()}
	khj, err := k.getJob(jobName, jobNamespace)
	if err != nil {
		return fmt.Errorf("Error when setting execution error on job %s %s %w", jobName, jobNamespace, err)
	}
	jobState, err := getJobState(khj)
	if err != nil {
		return fmt.Errorf("Error when setting execution error on job (getting job state for current UUID) %s %s %w", jobName, jobNamespace, err)
	}
	details.CurrentUUID = jobState.CurrentUUID
	log.Debugln("Setting execution state of job", jobName, "to", details.OK, details.Errors, details.CurrentUUID, details.GetKHWorkload())
	err = k.storeCheckState(jobName, jobNamespace, details)
	if err != nil {
		return fmt.Errorf("Was unable to write an execution error to the CRD status with error: %w", err)
	}
	return nil
}
func (k *Kuberhealthy) AddCheck(c *external.Checker) {
	k.Checks = append(k.Checks, c)
}
func (k *Kuberhealthy) Shutdown(doneChan chan struct{}) {
	if k.shutdownCtxFunc != nil {
		log.Infoln("shutdown: aborting control context")
		k.shutdownCtxFunc()  
	}
	time.Sleep(5)  
	log.Infoln("shutdown: stopping checks")
	k.StopChecks()  
	log.Infoln("shutdown: ready for main program shutdown")
	doneChan <- struct{}{}
}
func (k *Kuberhealthy) StopChecks() {
	log.Infoln("control:", len(k.Checks), "checks stopping...")
	if k.cancelChecksFunc != nil {
		k.cancelChecksFunc()
	}
	for _, c := range k.Checks {
		go func(c external.Checker) {
			log.Infoln("control: check", c.Name(), "stopping...")
			err := c.Shutdown()
			if err != nil {
				log.Errorln("control: ERROR stopping check", c.Name(), err)
			}
			k.wg.Done()
			log.Infoln("control: check", c.Name(), "stopped")
		}(*c)
	}
	log.Infoln("control: waiting for all checks to stop")
	k.wg.Wait()
	log.Infoln("control: all checks stopped.")
}
func (k *Kuberhealthy) Start(ctx context.Context) {
	go k.stateReflector.Start()
	if cfg.EnableInflux == true {
		k.configureInfluxForwarding()
	}
	go k.StartWebServer()
	maxUpdateInterval := time.Second * 10
	externalChecksUpdateChan := make(chan struct{}, 50)
	externalChecksUpdateChanLimited := make(chan struct{}, 50)
	go notifyChanLimiter(maxUpdateInterval, externalChecksUpdateChan, externalChecksUpdateChanLimited)
	go k.monitorExternalChecks(ctx, externalChecksUpdateChan)
	becameMasterChan := make(chan struct{}, 10)
	lostMasterChan := make(chan struct{}, 10)
	go k.masterMonitor(ctx, becameMasterChan, lostMasterChan)
	go k.monitorKHJobs(ctx)
	configReloadChan := make(chan struct{})
	go configReloadNotifier(ctx, configReloadChan)
	for {
		select {
		case <-ctx.Done():  
			log.Infoln("control: shutting down from context abort...")
			return
		case <-becameMasterChan:  
			log.Infoln("control: Became master. Reconfiguring and starting checks.")
			k.StartChecks(ctx)
			k.StartReaper(ctx)
		case <-lostMasterChan:  
			log.Infoln("control: Lost master. Stopping checks.")
			k.StopChecks()
			k.StopReaper()
		case <-externalChecksUpdateChanLimited:  
			log.Infoln("control: Witnessed a khcheck resource change...")
			if isMaster {
				log.Infoln("control: Reloading external check configurations due to khcheck update")
				k.RestartChecks(ctx)
				k.RestartReaper(ctx)
			}
		case <-configReloadChan:
			log.Infoln("control: Witnessed a kuberhealthy configuration change...")
			if isMaster {
				log.Infoln("control: Reloading external check configurations due to kuberhealthy configuration update")
				k.RestartChecks(ctx)
				k.RestartReaper(ctx)
			}
		}
	}
}
func (k *Kuberhealthy) StartReaper(ctx context.Context) {
	reaperCtx, reaperCtxCancel := context.WithCancel(ctx)
	k.cancelReaperFunc = reaperCtxCancel
	go reaper(reaperCtx)
}
func (k *Kuberhealthy) StopReaper() {
	if k.cancelReaperFunc != nil {
		k.cancelReaperFunc()
	}
}
func (k *Kuberhealthy) RestartReaper(ctx context.Context) {
	k.StopReaper()
	k.StartReaper(ctx)
}
func (k *Kuberhealthy) RestartChecks(ctx context.Context) {
	k.StopChecks()
	k.StartChecks(ctx)
}
func (k *Kuberhealthy) khStateResourceReaper(ctx context.Context) {
	ticker := time.NewTicker(time.Minute)
	defer ticker.Stop()
	log.Infoln("khState reaper: starting up")
	for {
		select {
		case <-ticker.C:
			log.Infoln("khState reaper: starting to run an audit")
			err := k.reapKHStateResources(ctx)
			if err != nil {
				log.Errorln("khState reaper: Error when reaping khState resources:", err)
			}
		case <-ctx.Done():
			log.Infoln("khState reaper: stopping")
			return
		}
	}
}