func (c *testContext) ConfigureHAProxy(loadBalancerAddress string, loadBalancerSSHPort int) {
	log.Info("Configuring H/A proxy...")
	ctx := context.TODO()
	keyFile := filepath.Join(c.tmpDir, "cluster-key")
	sshClient, err := ssh.NewClient(ssh.ClientParams{
		User:           "root",
		Host:           loadBalancerAddress,
		Port:           uint16(loadBalancerSSHPort),
		PrivateKeyPath: keyFile,
	})
	require.NoError(c.t, err)
	defer sshClient.Close()
	installer, err := capeios.Identify(ctx, sshClient)
	require.NoError(c.t, err)
	runner := &sudo.Runner{Runner: sshClient}
	cfg, err := envcfg.GetEnvSpecificConfig(ctx, installer.PkgType, "default", "", runner)
	require.NoError(c.t, err)
	baseResource := recipe.BuildBasePlan(installer.PkgType)
	dockerConfigResource, err := buildDockerConfigResource(c)
	require.NoError(c.t, err)
	criResource := recipe.BuildCRIPlan(
		ctx,
		&existinginfrav1.ContainerRuntime{
			Kind:    "docker",
			Package: "docker-ce",
			Version: "19.03.8",
		},
		cfg,
		installer.PkgType)
	ips := []string{"172.17.0.3", "172.17.0.4"}
	haConfigResource := &resource.File{
		Content:     generateHAConfiguration(ips),
		Destination: "/tmp/haproxy.cfg",
	}
	haproxyResource := &resource.Run{
		Script:     object.String("mkdir /tmp/haproxy && docker run --detach --name haproxy -v /tmp/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg -v /tmp/haproxy:/var/lib/haproxy -p 6443:6443 haproxy:2.3"),
		UndoScript: object.String("docker rm haproxy || true"),
	}
	lbPlanBuilder := plan.NewBuilder()
	lbPlanBuilder.AddResource("install:base", baseResource)
	lbPlanBuilder.AddResource("install:docker-repo-config", dockerConfigResource,
		plan.DependOn("install:base"))
	lbPlanBuilder.AddResource("install:cri", criResource, plan.DependOn("install:docker-repo-config"))
	lbPlanBuilder.AddResource("install:ha-config", haConfigResource, plan.DependOn("install:cri"))
	lbPlanBuilder.AddResource("install:haproxy", haproxyResource, plan.DependOn("install:ha-config"))
	lbPlan, err := lbPlanBuilder.Plan()
	require.NoError(c.t, err)
	err = lbPlan.Undo(ctx, runner, plan.EmptyState)
	require.NoError(c.t, err)
	_, err = lbPlan.Apply(ctx, runner, plan.EmptyDiff())
	require.NoError(c.t, err)
}
Create a load balancer so that we can repave machines
package test
import (
	"bytes"
	"context"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"testing"
	"time"
	log "github.com/sirupsen/logrus"
	"github.com/stretchr/testify/require"
	existinginfrav1 "github.com/weaveworks/cluster-api-provider-existinginfra/apis/cluster.weave.works/v1alpha3"
	capeios "github.com/weaveworks/cluster-api-provider-existinginfra/pkg/apis/wksprovider/machine/os"
	"github.com/weaveworks/cluster-api-provider-existinginfra/pkg/plan"
	"github.com/weaveworks/cluster-api-provider-existinginfra/pkg/plan/recipe"
	"github.com/weaveworks/cluster-api-provider-existinginfra/pkg/plan/resource"
	"github.com/weaveworks/cluster-api-provider-existinginfra/pkg/plan/runners/ssh"
	"github.com/weaveworks/cluster-api-provider-existinginfra/pkg/plan/runners/sudo"
	"github.com/weaveworks/cluster-api-provider-existinginfra/pkg/specs"
	"github.com/weaveworks/cluster-api-provider-existinginfra/pkg/utilities/envcfg"
	"github.com/weaveworks/cluster-api-provider-existinginfra/pkg/utilities/object"
)
type testContext struct {
	t       *testing.T
	tmpDir  string
	testDir string
}
type commandConfig struct {
	CheckError bool
	Dir        string
	Env        []string
	Stdout     *os.File
	Stderr     *os.File
}
func getTestContext(t *testing.T) *testContext {
	tmpDir, err := ioutil.TempDir("", "tmp_dir")
	require.NoError(t, err)
	return getTestContextFrom(t, tmpDir)
}
func getTestContextFrom(t *testing.T, tmpDir string) *testContext {
	log.Infof("Using temporary directory: %s\n", tmpDir)
	c := &testContext{
		t:       t,
		tmpDir:  tmpDir,
		testDir: getTestDir(t),
	}
	log.Info("Installing kind...")
	c.runWithConfig(commandConfig{CheckError: true, Env: env("GO111MODULE=on", "HOME="+tmpDir, "CGO_ENABLED=0"), Stdout: os.Stdout, Stderr: os.Stderr},
		"go", "get", "sigs.k8s.io/kind@v0.9.0")
	log.Info("Installing footloose...")
	c.runWithConfig(commandConfig{CheckError: true, Env: env("GO111MODULE=on", "HOME="+tmpDir, "CGO_ENABLED=0"), Stdout: os.Stdout, Stderr: os.Stderr},
		"go", "get", "github.com/weaveworks/footloose")
	log.Info("Installing clusterctl...")
	for {
		fetchCmd := fmt.Sprintf("curl -L https://github.com/kubernetes-sigs/cluster-api/releases/download/v0.3.10/clusterctl-%s-%s -o %s/clusterctl && chmod a+x %s/clusterctl",
			c.getOS(), c.getArch(), c.tmpDir, c.tmpDir)
		c.runAndCheckError("sh", "-c", fetchCmd)
		_, _, err := c.runCollectingOutput(filepath.Join(c.tmpDir, "clusterctl"), "version")
		if err == nil {
			break
		}
	}
	err := os.Setenv("HOME", c.tmpDir)
	require.NoError(c.t, err)
	kindCmd := filepath.Join(c.tmpDir, "go", "bin", "kind")
	c.run(kindCmd, "delete", "cluster")
	c.runAndCheckError(kindCmd, "create", "cluster")
	return c
}
func (c *testContext) cleanup() {
	log.Infof("About to remove temp dir: '%s'", c.tmpDir)
	c.runAndCheckError(filepath.Join(c.tmpDir, "go", "bin", "kind"), "delete", "cluster")
	os.RemoveAll(c.tmpDir)
}
func (c *testContext) getOS() string {
	osbytes, _, err := c.runCollectingOutput("uname", "-s")
	require.NoError(c.t, err)
	os := string(osbytes)
	switch {
	case strings.HasPrefix(os, "Linux"):
		return "linux"
	case strings.HasPrefix(os, "Darwin"):
		return "darwin"
	default:
		require.FailNow(c.t, fmt.Sprintf("Unknown operating system: '%s'", os))
	}
	return ""
}
var archmap = map[string]string{
	"armv5":   "armv5",
	"armv6":   "armv6",
	"armv7":   "armv7",
	"aarch64": "arm64",
	"x86_64":  "amd64",
	"x86":     "386",
	"i386":    "386",
	"i686":    "386",
}
func (c *testContext) getArch() string {
	archbytes, _, err := c.runCollectingOutput("uname", "-m")
	arch := string(archbytes)
	require.NoError(c.t, err)
	for key, val := range archmap {
		if strings.HasPrefix(arch, key) {
			return val
		}
	}
	require.FailNow(c.t, fmt.Sprintf("Unknown architecture: '%s'", arch))
	return ""
}
func getTestDir(t *testing.T) string {
	testDir, err := os.Getwd()
	require.NoError(t, err)
	return testDir
}
func env(items ...string) []string {
	return items
}
func (c *testContext) applyManagementManifest(manifest string) {
	f, err := ioutil.TempFile(c.tmpDir, "---manifest--*---")
	require.NoError(c.t, err)
	defer os.Remove(f.Name())
	err = ioutil.WriteFile(f.Name(), []byte(manifest), 0600)
	require.NoError(c.t, err)
	c.runAndCheckError("kubectl", "apply", "-f", f.Name())
}
func (c *testContext) applyWorkloadManifest(manifest, kubeconfig string) {
	f, err := ioutil.TempFile(c.tmpDir, "---manifest--*---")
	require.NoError(c.t, err)
	defer os.Remove(f.Name())
	err = ioutil.WriteFile(f.Name(), []byte(manifest), 0600)
	require.NoError(c.t, err)
	_, eout, err := c.runCollectingOutputWithConfig(commandConfig{Env: env("KUBECONFIG=" + kubeconfig)},
		"kubectl", "apply", "-f", f.Name())
	if err != nil {
		require.FailNow(c.t, fmt.Sprintf("Failed to apply manifest: %v -- %s", err, eout))
	}
}
func (c *testContext) run(cmdItems ...string) {
	c.runWithConfig(commandConfig{Stdout: os.Stdout, Stderr: os.Stderr, Env: os.Environ()}, cmdItems...)
}
func (c *testContext) runAndCheckError(cmdItems ...string) {
	c.runWithConfig(commandConfig{Stdout: os.Stdout, Stderr: os.Stderr, Env: os.Environ(), CheckError: true}, cmdItems...)
}
func (c *testContext) runWithConfig(config commandConfig, cmdItems ...string) {
	c.t.Helper()
	cmd := exec.Command(cmdItems[0], cmdItems[1:]...)
	cmd.Dir = config.Dir
	cmd.Stdout = config.Stdout
	cmd.Stderr = config.Stderr
	cmd.Env = config.Env
	err := cmd.Run()
	if config.CheckError {
		if err != nil {
			log.Errorf("Run error: %v", err)
		}
		require.NoError(c.t, err)
	}
}
func (c *testContext) runCollectingOutput(cmdItems ...string) ([]byte, []byte, error) {
	return c.runCollectingOutputWithConfig(commandConfig{Env: os.Environ()}, cmdItems...)
}
func (c *testContext) runCollectingOutputWithConfig(config commandConfig, cmdItems ...string) ([]byte, []byte, error) {
	cmd := exec.Command(cmdItems[0], cmdItems[1:]...)
	var stdout, stderr bytes.Buffer
	cmd.Dir = config.Dir
	cmd.Env = config.Env
	cmd.Stdout = &stdout
	cmd.Stderr = &stderr
	err := cmd.Run()
	return stdout.Bytes(), stderr.Bytes(), err
}
func (c *testContext) sshCall(ip, port, cmd string) ([]byte, []byte, error) {
	return c.runCollectingOutput("ssh", "-i", filepath.Join(c.tmpDir, "cluster-key"), "-l", "root",
		"-o", "UserKnownHostsFile /dev/null", "-o", "StrictHostKeyChecking=no", "-p", port, ip, cmd)
}
func (c *testContext) sshAction(ip, port, cmd string) {
	c.runWithConfig(commandConfig{Stdout: os.Stdout, Stderr: os.Stderr, Env: os.Environ(), CheckError: false},
		"ssh", "-i", filepath.Join(c.tmpDir, "cluster-key"), "-l", "root",
		"-o", "UserKnownHostsFile /dev/null", "-o", "StrictHostKeyChecking=no", "-p", port, ip, cmd)
}
func (c *testContext) makeSSHCallAndCheckError(ip, port, cmd string) {
	c.runAndCheckError("ssh", "-i", filepath.Join(c.tmpDir, "cluster-key"), "-l", "root",
		"-o", "UserKnownHostsFile /dev/null", "-o", "StrictHostKeyChecking=no", "-p", port, ip, cmd)
}
func (c *testContext) makeSSHCallWithRetries(ip, port, cmd string, retryCount int) {
	for ; retryCount > 0; retryCount-- {
		out, eout, err := c.sshCall(ip, port, cmd)
		if err == nil {
			return
		}
		log.Infof("Call '%s' failed: %s, %s, %v", cmd, out, eout, err)
		time.Sleep(30 * time.Second)
	}
	require.FailNow(c.t, fmt.Sprintf("SSH call failed all retries"))
}
func (c *testContext) makeSSHCallWithFailureHandler(ip, port, cmd string, failfunc func(), retryCount int) {
	for ; retryCount > 0; retryCount-- {
		out, eout, err := c.sshCall(ip, port, cmd)
		if err == nil {
			return
		}
		log.Infof("Call '%s' failed: %s, %s, %v", cmd, out, eout, err)
		time.Sleep(30 * time.Second)
	}
	failfunc()
	require.FailNow(c.t, fmt.Sprintf("SSH call failed all retries"))
}
func (c *testContext) ensureCount(itemType string, count int, kubeconfigPath string) {
	for retryCount := 1; retryCount <= 30; retryCount++ {
		cmdItems := []string{"kubectl", "get", itemType, "--all-namespaces", "--no-headers=true"}
		cmdResults, eout, err := c.runCollectingOutputWithConfig(commandConfig{Env: env("KUBECONFIG=" + kubeconfigPath)}, cmdItems...)
		if string(eout) != "" {
			log.Infof("EOUT: %s, ERR: %v", eout, err)
		}
		if len(strings.Split(string(cmdResults), "\n")) > count {  
			return
		}
		log.Infof("Waiting for %d %s, retry: %d...", count, itemType, retryCount)
		c.runWithConfig(commandConfig{Env: env("KUBECONFIG=" + kubeconfigPath)},
			"sh", "-c", "kubectl logs $(kubectl get pods -A | grep wks-controller | awk '{print($2)}') -n test")
		time.Sleep(30 * time.Second)
	}
	require.FailNow(c.t, fmt.Sprintf("Fewer than %d %s are running...", count, itemType))
}
func (c *testContext) ensureRunning(itemType, kubeconfigPath string) {
	cmdItems := []string{"kubectl", "get", itemType, "--all-namespaces", "-o",
		`jsonpath={range .items[*]}{"\n"}{@.metadata.name}:{@.spec.unschedulable}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}`}
	for retryCount := 1; retryCount <= 30; retryCount++ {
		allReady := true
		cmdResults, _, _ := c.runCollectingOutputWithConfig(commandConfig{Env: env("KUBECONFIG=" + kubeconfigPath)}, cmdItems...)
		strs := strings.Split(string(cmdResults), "\n")
		for _, str := range strs {
			if str != "" && (!strings.Contains(str, "Ready=True") || strings.Contains(str, ":true:")) {
				log.Infof("Waiting for: %s", str)
				allReady = false
			}
		}
		if allReady {
			return
		}
		log.Infof("Waiting for all %s to be running, retry: %d...", itemType, retryCount)
		if retryCount > 6 {
			c.runWithConfig(commandConfig{Env: env("KUBECONFIG=" + kubeconfigPath), Stdout: os.Stdout, Stderr: os.Stderr},
				"kubectl", "get", "nodes", "-A", "-o", "yaml")
		}
		time.Sleep(30 * time.Second)
	}
	require.FailNow(c.t, fmt.Sprintf("Not all %s are running...", itemType))
}
func (c *testContext) ensureAllStoppedRunning(itemType, kubeconfigPath string) {
	cmdItems := []string{"kubectl", "get", itemType, "--all-namespaces", "-o",
		`jsonpath={range .items[*]}{"\n"}{@.metadata.name}:{@.spec.unschedulable}:{range @.status.conditions[*]}{@.type}={@.status};{end}{end}`}
	unreadySeen := ""
	for retryCount := 1; retryCount <= 100; retryCount++ {
		cmdResults, _, err := c.runCollectingOutputWithConfig(commandConfig{Env: env("KUBECONFIG=" + kubeconfigPath)}, cmdItems...)
		if err != nil {
			time.Sleep(time.Second * 5)
			continue
		}
		strs := strings.Split(string(cmdResults), "\n")
		for _, str := range strs {
			nameAndConditions := strings.Split(str, ":")
			log.Infof("Response: %s", str)
			if len(nameAndConditions) < 3 {  
				continue
			}
			if str != "" && nameAndConditions[0] != unreadySeen &&
				(!strings.Contains(nameAndConditions[2], "Ready=True") || nameAndConditions[1] == "true") {
				log.Infof("Machine %s stopped...", nameAndConditions[0])
				if unreadySeen != "" {
					return
				}
				unreadySeen = nameAndConditions[0]
			}
		}
		if retryCount%10 == 0 {
			log.Infof("Waiting for all %s to have stopped running, retry: %d...", itemType, retryCount)
		}
		time.Sleep(5 * time.Second)
	}
	require.FailNow(c.t, fmt.Sprintf("Some %s did not stop running...", itemType))
}
func (c *testContext) getProviderEnvironment() []string {
	tag, _, err := c.runCollectingOutput(filepath.Join(c.testDir, "../../../tools/image-tag"))
	require.NoError(c.t, err)
	return env("NAMESPACE=test", "CONTROL_PLANE_MACHINE_COUNT=2", "WORKER_MACHINE_COUNT=0", "HOME="+c.tmpDir,
		"CONTROL_PLANE_ENDPOINT=172.17.0.2:6443",
		"EXISTINGINFRA_CONTROLLER_IMAGE=docker.io/weaveworks/cluster-api-existinginfra-controller:"+string(tag))
}