func (h *Harness) Stop() {
	h.T.Log("cleaning up")
	if h.managerStopCh != nil {
		close(h.managerStopCh)
		h.managerStopCh = nil
	}
	if h.kind != nil {
		logDir := filepath.Join(h.TestSuite.ArtifactsDir, fmt.Sprintf("kind-logs-%d", time.Now().Unix()))
		h.T.Log("collecting cluster logs to", logDir)
		if err := h.kind.CollectLogs(logDir); err != nil {
			h.T.Log("error collecting kind cluster logs", err)
		}
	}
	if h.bgProcesses != nil {
		for _, p := range h.bgProcesses {
			h.T.Logf("killing process %q", p)
			err := p.Process.Kill()
			if err != nil {
				h.T.Logf("bg process: %q kill error %v", p, err)
			}
			ps, err := p.Process.Wait()
			if err != nil {
				h.T.Logf("bg process: %q kill wait error %v", p, err)
			}
			if ps != nil {
				h.T.Logf("bg process: %q exit code %v", p, ps.ExitCode())
			}
		}
	}
	if h.TestSuite.SkipClusterDelete {
		cwd, err := os.Getwd()
		if err != nil {
			h.T.Logf("issue getting work directory %v", err)
		}
		kubeconfig := filepath.Join(cwd, "kubeconfig")
		h.T.Log("skipping cluster tear down")
		h.T.Logf("to connect to the cluster, run: export KUBECONFIG=\"%s\"", kubeconfig)
		return
	}
	if h.env != nil {
		h.T.Log("tearing down mock control plane")
		if err := h.env.Stop(); err != nil {
			h.T.Log("error tearing down mock control plane", err)
		}
		h.env = nil
	}
	h.T.Logf("removing temp folder: %q", h.tempPath)
	if err := os.RemoveAll(h.tempPath); err != nil {
		h.T.Log("error removing temporary directory", err)
	}
	if h.kind != nil {
		h.T.Log("tearing down kind cluster")
		if err := h.kind.Stop(); err != nil {
			h.T.Log("error tearing down kind cluster", err)
		}
		h.kind = nil
	}
	h.Report()
}
Stop the test environment and clean up the harness.
package test
import (
	"bytes"
	"context"
	"errors"
	"fmt"
	"io/ioutil"
	"math/rand"
	"os"
	"os/exec"
	"os/signal"
	"path/filepath"
	"strings"
	"sync"
	"testing"
	"time"
	volumetypes "github.com/docker/docker/api/types/volume"
	docker "github.com/docker/docker/client"
	"gopkg.in/yaml.v2"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/client-go/discovery"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"
	"sigs.k8s.io/controller-runtime/pkg/client"
	"sigs.k8s.io/controller-runtime/pkg/client/config"
	"sigs.k8s.io/controller-runtime/pkg/envtest"
	kindConfig "sigs.k8s.io/kind/pkg/apis/config/v1alpha4"
	harness "github.com/kudobuilder/kuttl/pkg/apis/testharness/v1beta1"
	"github.com/kudobuilder/kuttl/pkg/file"
	"github.com/kudobuilder/kuttl/pkg/http"
	"github.com/kudobuilder/kuttl/pkg/report"
	testutils "github.com/kudobuilder/kuttl/pkg/test/utils"
)
type Harness struct {
	TestSuite harness.TestSuite
	T         *testing.T
	logger        testutils.Logger
	managerStopCh chan struct{}
	config        *rest.Config
	docker        testutils.DockerClient
	client        client.Client
	dclient       discovery.DiscoveryInterface
	env           *envtest.Environment
	kind          *kind
	tempPath      string
	clientLock    sync.Mutex
	configLock    sync.Mutex
	stopping      bool
	bgProcesses   []*exec.Cmd
	report        *report.Testsuites
}
func (h *Harness) LoadTests(dir string) ([]*Case, error) {
	dir, err := filepath.Abs(dir)
	if err != nil {
		return nil, err
	}
	files, err := ioutil.ReadDir(dir)
	if err != nil {
		return nil, err
	}
	var tests []*Case
	timeout := h.GetTimeout()
	h.T.Logf("going to run test suite with timeout of %d seconds for each step", timeout)
	for _, file := range files {
		if !file.IsDir() {
			continue
		}
		tests = append(tests, &Case{
			Timeout:            timeout,
			Steps:              []*Step{},
			Name:               file.Name(),
			PreferredNamespace: h.TestSuite.Namespace,
			Dir:                filepath.Join(dir, file.Name()),
			SkipDelete:         h.TestSuite.SkipDelete,
			Suppress:           h.TestSuite.Suppress,
		})
	}
	return tests, nil
}
func (h *Harness) GetLogger() testutils.Logger {
	if h.logger == nil {
		h.logger = testutils.NewTestLogger(h.T, "")
	}
	return h.logger
}
func (h *Harness) GetTimeout() int {
	timeout := 30
	if h.TestSuite.Timeout != 0 {
		timeout = h.TestSuite.Timeout
	}
	return timeout
}
func (h *Harness) RunKIND() (*rest.Config, error) {
	if h.kind == nil {
		var err error
		err = h.initTempPath()
		if err != nil {
			return nil, err
		}
		kind := newKind(h.TestSuite.KINDContext, h.kubeconfigPath(), h.GetLogger())
		h.kind = &kind
		if h.kind.IsRunning() {
			h.kind = nil
			msg := "KIND is already running, unable to start"
			h.T.Log(msg)
			return nil, errors.New(msg)
		}
		kindCfg := &kindConfig.Cluster{}
		if h.TestSuite.KINDConfig != "" {
			h.T.Logf("Loading KIND config from %s", h.TestSuite.KINDConfig)
			var err error
			kindCfg, err = h.loadKindConfig(h.TestSuite.KINDConfig)
			if err != nil {
				return nil, err
			}
		}
		dockerClient, err := h.DockerClient()
		if err != nil {
			return nil, err
		}
		dockerClient.NegotiateAPIVersion(context.TODO())
		h.addNodeCaches(dockerClient, kindCfg)
		h.T.Log("Starting KIND cluster")
		if err := h.kind.Run(kindCfg); err != nil {
			return nil, err
		}
		if err := h.kind.AddContainers(dockerClient, h.TestSuite.KINDContainers, h.T); err != nil {
			return nil, err
		}
	}
	return clientcmd.BuildConfigFromFlags("", h.kubeconfigPath())
}
func (h *Harness) initTempPath() (err error) {
	if h.tempPath == "" {
		h.tempPath, err = ioutil.TempDir("", "kuttl")
		h.T.Log("temp folder created", h.tempPath)
	}
	return err
}
func (h *Harness) addNodeCaches(dockerClient testutils.DockerClient, kindCfg *kindConfig.Cluster) {
	if !h.TestSuite.KINDNodeCache {
		return
	}
	if len(kindCfg.Nodes) == 0 {
		kindCfg.Nodes = append(kindCfg.Nodes, kindConfig.Node{})
	}
	if h.TestSuite.KINDContext == "" {
		h.TestSuite.KINDContext = harness.DefaultKINDContext
	}
	for index := range kindCfg.Nodes {
		volume, err := dockerClient.VolumeCreate(context.TODO(), volumetypes.VolumeCreateBody{
			Driver: "local",
			Name:   fmt.Sprintf("%s-%d", h.TestSuite.KINDContext, index),
		})
		if err != nil {
			h.T.Log("error creating volume for node", err)
			continue
		}
		h.T.Log("node mount point", volume.Mountpoint)
		kindCfg.Nodes[index].ExtraMounts = append(kindCfg.Nodes[index].ExtraMounts, kindConfig.Mount{
			ContainerPath: "/var/lib/containerd",
			HostPath:      volume.Mountpoint,
		})
	}
}
func (h *Harness) RunTestEnv() (*rest.Config, error) {
	started := time.Now()
	testenv, err := testutils.StartTestEnvironment(h.TestSuite.ControlPlaneArgs, h.TestSuite.AttachControlPlaneOutput)
	if err != nil {
		return nil, err
	}
	h.T.Logf("started test environment (kube-apiserver and etcd) in %v, with following options:\n%s",
		time.Since(started),
		strings.Join(testenv.Environment.KubeAPIServerFlags, "\n"))
	h.env = testenv.Environment
	return testenv.Config, nil
}
func (h *Harness) Config() (*rest.Config, error) {
	h.configLock.Lock()
	defer h.configLock.Unlock()
	if h.config != nil {
		return h.config, nil
	}
	var err error
	switch {
	case h.TestSuite.StartControlPlane:
		h.T.Log("running tests with a mocked control plane (kube-apiserver and etcd).")
		h.config, err = h.RunTestEnv()
	case h.TestSuite.StartKIND:
		h.T.Log("running tests with KIND.")
		h.config, err = h.RunKIND()
	default:
		h.T.Log("running tests using configured kubeconfig.")
		h.config, err = config.GetConfig()
		if err != nil {
			return nil, err
		}
		h.config.WarningHandler = rest.NewWarningWriter(os.Stderr, rest.WarningWriterOptions{Deduplicate: true})
		inCluster, err := testutils.InClusterConfig()
		if err != nil {
			return nil, err
		}
		if inCluster {
			return h.config, nil
		}
	}
	if err != nil {
		return h.config, err
	}
	if !h.TestSuite.StartControlPlane {
		err = testutils.WaitForSA(h.config, "default", "default")
		if err != nil {
			if h.TestSuite.Namespace != "" {
				tempErr := testutils.WaitForSA(h.config, "default", h.TestSuite.Namespace)
				if tempErr != nil {
					return h.config, err
				}
			} else {
				return h.config, err
			}
		}
		h.T.Logf("Successful connection to cluster at: %s", h.config.Host)
	}
	f, err := os.Create("kubeconfig")
	if err != nil {
		return h.config, err
	}
	defer f.Close()
	return h.config, testutils.Kubeconfig(h.config, f)
}
func (h *Harness) Client(forceNew bool) (client.Client, error) {
	h.clientLock.Lock()
	defer h.clientLock.Unlock()
	if h.client != nil && !forceNew {
		return h.client, nil
	}
	cfg, err := h.Config()
	if err != nil {
		return nil, err
	}
	h.client, err = testutils.NewRetryClient(cfg, client.Options{
		Scheme: testutils.Scheme(),
	})
	return h.client, err
}
func (h *Harness) DiscoveryClient() (discovery.DiscoveryInterface, error) {
	h.clientLock.Lock()
	defer h.clientLock.Unlock()
	if h.dclient != nil {
		return h.dclient, nil
	}
	cfg, err := h.Config()
	if err != nil {
		return nil, err
	}
	h.dclient, err = discovery.NewDiscoveryClientForConfig(cfg)
	return h.dclient, err
}
func (h *Harness) DockerClient() (testutils.DockerClient, error) {
	if h.docker != nil {
		return h.docker, nil
	}
	var err error
	h.docker, err = docker.NewClientWithOpts(docker.FromEnv)
	return h.docker, err
}
func (h *Harness) RunTests() {
	defer h.Stop()
	h.T.Log("running tests")
	testDirs := h.testPreProcessing()
	realTestSuite := make(map[string][]*Case)
	for _, testDir := range testDirs {
		tempTests, err := h.LoadTests(testDir)
		if err != nil {
			h.T.Fatal(err)
		}
		h.T.Logf("testsuite: %s has %d tests", testDir, len(tempTests))
		realTestSuite[testDir] = tempTests
	}
	h.T.Run("harness", func(t *testing.T) {
		for testDir, tests := range realTestSuite {
			suite := h.report.NewSuite(testDir)
			for _, test := range tests {
				test := test
				test.Client = h.Client
				test.DiscoveryClient = h.DiscoveryClient
				t.Run(test.Name, func(t *testing.T) {
					t.Parallel()
					test.Logger = testutils.NewTestLogger(t, test.Name)
					if err := test.LoadTestSteps(); err != nil {
						t.Fatal(err)
					}
					tc := report.NewCase(test.Name)
					test.Run(t, tc)
					suite.AddTestcase(tc)
				})
			}
		}
	})
	h.T.Log("run tests finished")
}
func (h *Harness) testPreProcessing() []string {
	testDirs := []string{}
	for _, dir := range h.TestSuite.TestDirs {
		if http.IsURL(dir) {
			err := h.initTempPath()
			if err != nil {
				h.T.Fatal(err)
			}
			client := http.NewClient()
			h.T.Logf("downloading %s", dir)
			folder, err := ioutil.TempDir(h.tempPath, filepath.Base(dir))
			if err != nil {
				h.T.Fatal(err)
			}
			filePath, err := client.DownloadFile(dir, folder)
			if err != nil {
				h.T.Fatal(err)
			}
			err = file.UntarInPlace(filePath)
			if err != nil {
				h.T.Fatal(err)
			}
			testDirs = append(testDirs, file.TrimExt(filePath))
		} else {
			testDirs = append(testDirs, dir)
		}
	}
	return testDirs
}
func (h *Harness) Run() {
	go func() {
		sigchan := make(chan os.Signal, 1)
		signal.Notify(sigchan, os.Interrupt)
		sig := <-sigchan
		h.Stop()
		h.T.Log("failed with", sig)
		os.Exit(-1)
	}()
	h.Setup()
	h.RunTests()
}
func (h *Harness) Setup() {
	rand.Seed(time.Now().UTC().UnixNano())
	h.report = report.NewSuiteCollection(h.TestSuite.Name)
	h.T.Log("starting setup")
	cl, err := h.Client(false)
	if err != nil {
		h.fatal(fmt.Errorf("fatal error getting client: %v", err))
	}
	dClient, err := h.DiscoveryClient()
	if err != nil {
		h.fatal(fmt.Errorf("fatal error getting discovery client: %v", err))
	}
	crdKinds := []runtime.Object{
		testutils.NewResource("apiextensions.k8s.io/v1", "CustomResourceDefinition", "", ""),
		testutils.NewResource("apiextensions.k8s.io/v1beta1", "CustomResourceDefinition", "", ""),
	}
	crds, err := testutils.InstallManifests(context.TODO(), cl, dClient, h.TestSuite.CRDDir, crdKinds...)
	if err != nil {
		h.fatal(fmt.Errorf("fatal error installing crds: %v", err))
	}
	if err := envtest.WaitForCRDs(h.config, crds, envtest.CRDInstallOptions{
		PollInterval: 100 * time.Millisecond,
		MaxTime:      10 * time.Second,
	}); err != nil {
		h.fatal(fmt.Errorf("fatal error waiting for crds: %v", err))
	}
	cl, err = h.Client(true)
	if err != nil {
		h.fatal(fmt.Errorf("fatal error getting client after crd update: %v", err))
	}
	for _, manifestDir := range h.TestSuite.ManifestDirs {
		if _, err := testutils.InstallManifests(context.TODO(), cl, dClient, manifestDir); err != nil {
			h.fatal(fmt.Errorf("fatal error installing manifests: %v", err))
		}
	}
	bgs, err := testutils.RunCommands(context.TODO(), h.GetLogger(), "default", h.TestSuite.Commands, "", h.TestSuite.Timeout, "")
	h.bgProcesses = append(h.bgProcesses, bgs...)
	if err != nil {
		h.fatal(fmt.Errorf("fatal error running commands: %v", err))
	}
}