func (i *Instance) WaitForExpected(expectedIPs []string) (err error) {
	_, err = waitForLocalGSLBNew(i.w.t, i.w.state.gslb.host, i.w.state.gslb.port, expectedIPs)
	if err != nil {
		fmt.Println(i.GetStatus(fmt.Sprintf("expected IPs: %s", expectedIPs)).String())
	}
	return
}
WaitForExpected waits until GSLB dig doesnt return list of expected IP's
package utils
import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"path/filepath"
	"strconv"
	"strings"
	"testing"
	"time"
	"github.com/AbsaOSS/gopkg/dns"
	gopkgstr "github.com/AbsaOSS/gopkg/strings"
	"github.com/gruntwork-io/terratest/modules/helm"
	"github.com/gruntwork-io/terratest/modules/k8s"
	"github.com/gruntwork-io/terratest/modules/random"
	"github.com/gruntwork-io/terratest/modules/shell"
	"github.com/stretchr/testify/require"
	"gopkg.in/yaml.v2"
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
type Workflow struct {
	error      error
	namespace  string
	cluster    string
	k8sOptions *k8s.KubectlOptions
	t          *testing.T
	settings   struct {
		ingressResourcePath string
		gslbResourcePath    string
		ingressName         string
	}
	state struct {
		namespaceCreated bool
		testApp          struct {
			name        string
			message     string
			isRunning   bool
			isInstalled bool
		}
		gslb struct {
			name        string
			host        string
			port        int
			isInstalled bool
		}
	}
}
type Instance struct {
	w *Workflow
}
type TestAppResult struct {
	Message string `json:"message"`
	Version string `json:"version"`
	Color   string `json:"color"`
	Pod     string `json:"hostname"`
	Body    string
}
type InstanceStatus struct {
	Annotation       string   `json:"annotation"`
	AppMessage       string   `json:"app-msg"`
	AppRunning       bool     `json:"podinfo-running"`
	AppReplicas      string   `json:"podinfo-replicas"`
	LocalTargets     []string `json:"local-targets-ip"`
	Ingresses        []string `json:"ingress-ip"`
	Dig              []string `json:"dig-result"`
	CoreDNS          string   `json:"coredns-ip"`
	GslbHealthStatus string   `json:"gslb-status"`
	Cluster          string   `json:"cluster"`
	Namespace        string   `json:"namespace"`
	Endpoint0DNSName string   `json:"ep0-dns-name"`
	Endpoint0Targets string   `json:"ep0-dns-targets"`
	Endpoint1DNSName string   `json:"ep1-dns-name"`
	Endpoint1Targets string   `json:"ep1-dns-targets"`
}
func NewWorkflow(t *testing.T, cluster string, port int) *Workflow {
	var err error
	if cluster == "" {
		err = fmt.Errorf("empty cluster")
	}
	if port < 1000 {
		err = fmt.Errorf("invalid port")
	}
	w := new(Workflow)
	w.cluster = cluster
	w.namespace = fmt.Sprintf("k8gb-test-%s", strings.ToLower(random.UniqueId()))
	w.k8sOptions = k8s.NewKubectlOptions(cluster, "", w.namespace)
	w.t = t
	w.state.gslb.port = port
	w.error = err
	return w
}
func (w *Workflow) WithIngress(path string) *Workflow {
	if path == "" {
		w.error = fmt.Errorf("empty ingress resource path")
	}
	w.settings.ingressResourcePath = path
	return w
}
func (w *Workflow) WithGslb(path, host string) *Workflow {
	var err error
	if host == "" {
		w.error = fmt.Errorf("empty gslb host")
	}
	if path == "" {
		w.error = fmt.Errorf("empty gslb resource path")
	}
	w.settings.gslbResourcePath, err = filepath.Abs(path)
	if err != nil {
		w.error = fmt.Errorf("reading %s; %s", path, err)
	}
	w.state.gslb.name, err = w.getManifestName(w.settings.gslbResourcePath)
	if err != nil {
		w.error = err
	}
	w.state.gslb.host = host
	if err != nil {
		w.error = err
	}
	return w
}
func (w *Workflow) WithTestApp(uiMessage string) *Workflow {
	w.state.testApp.isInstalled = true
	w.state.testApp.name = "frontend-podinfo"
	w.state.testApp.message = uiMessage
	return w
}
func (w *Workflow) Start() (*Instance, error) {
	if w.error != nil {
		return nil, w.error
	}
	w.t.Logf("Create namespace %s", w.namespace)
	k8s.CreateNamespace(w.t, w.k8sOptions, w.namespace)
	w.state.namespaceCreated = true
	if w.settings.gslbResourcePath != "" {
		w.t.Logf("Create ingress %s from %s", w.state.gslb.name, w.settings.gslbResourcePath)
		k8s.KubectlApply(w.t, w.k8sOptions, w.settings.gslbResourcePath)
		k8s.WaitUntilIngressAvailable(w.t, w.k8sOptions, w.state.gslb.name, 60, 1*time.Second)
		ingress := k8s.GetIngress(w.t, w.k8sOptions, w.state.gslb.name)
		require.Equal(w.t, ingress.Name, w.state.gslb.name)
		w.settings.ingressName = w.state.gslb.name
	}
	if w.settings.ingressResourcePath != "" {
	}
	if w.state.testApp.isInstalled {
		const app = "https://stefanprodan.github.io/podinfo"
		w.t.Logf("Create test application %s", app)
		helmRepoAdd := shell.Command{
			Command: "helm",
			Args:    []string{"repo", "add", "--force-update", "podinfo", app},
		}
		helmRepoUpdate := shell.Command{
			Command: "helm",
			Args:    []string{"repo", "update"},
		}
		shell.RunCommand(w.t, helmRepoAdd)
		shell.RunCommand(w.t, helmRepoUpdate)
		helmOptions := helm.Options{
			KubectlOptions: w.k8sOptions,
			Version:        "5.1.1",
			SetValues:      map[string]string{"ui.message": w.state.testApp.message},
		}
		helm.Install(w.t, &helmOptions, "podinfo/podinfo", "frontend")
		testAppFilter := metav1.ListOptions{
			LabelSelector: "app.kubernetes.io/name=" + w.state.testApp.name,
		}
		k8s.WaitUntilNumPodsCreated(w.t, w.k8sOptions, testAppFilter, 1, 60, 1*time.Second)
		var testAppPods []corev1.Pod
		testAppPods = k8s.ListPods(w.t, w.k8sOptions, testAppFilter)
		for _, pod := range testAppPods {
			k8s.WaitUntilPodAvailable(w.t, w.k8sOptions, pod.Name, 60, 1*time.Second)
		}
		k8s.WaitUntilServiceAvailable(w.t, w.k8sOptions, w.state.testApp.name, 60, 1*time.Second)
		w.state.testApp.isRunning = true
	}
	return &Instance{
		w: w,
	}, nil
}
func (w *Workflow) getManifestName(path string) (string, error) {
	m := struct {
		Metadata struct {
			Name string `yaml:"name"`
		} `yaml:"metadata"`
	}{}
	yamlFile, err := ioutil.ReadFile(path)
	if err != nil {
		return "", fmt.Errorf("parse %s; %s", path, err)
	}
	err = yaml.Unmarshal(yamlFile, &m)
	if err != nil {
		return "", fmt.Errorf("unmarshall %s; %s", path, err)
	}
	return m.Metadata.Name, nil
}
func (i *Instance) Kill() {
	i.w.t.Logf("killing %s", i)
	if i.w.state.namespaceCreated {
		k8s.DeleteNamespace(i.w.t, i.w.k8sOptions, i.w.namespace)
	}
}
func (i *Instance) GetCoreDNSIP() string {
	cmd := shell.Command{
		Command: "kubectl",
		Args:    []string{"--context", i.w.k8sOptions.ContextName, "-n", "k8gb", "get", "svc", "k8gb-coredns", "--no-headers", "-o", "custom-columns=IP:spec.clusterIPs[0]"},
		Env:     i.w.k8sOptions.Env,
	}
	out, err := shell.RunCommandAndGetOutputE(i.w.t, cmd)
	require.NoError(i.w.t, err)
	require.NotEqual(i.w.t, "<none>", out)
	return out
}
func (i *Instance) GetIngressIPs() []string {
	var ingressIPs []string
	ingress := k8s.GetIngress(i.w.t, i.w.k8sOptions, i.w.settings.ingressName)
	for _, ip := range ingress.Status.LoadBalancer.Ingress {
		ingressIPs = append(ingressIPs, ip.IP)
	}
	return ingressIPs
}
func (i *Instance) StopTestApp() {
	k8s.RunKubectl(i.w.t, i.w.k8sOptions, "scale", "deploy", i.w.state.testApp.name, "--replicas=0")
	AssertGslbStatus(i.w.t, i.w.k8sOptions, i.w.state.gslb.name, i.w.state.gslb.host+":Unhealthy")
	i.w.state.testApp.isRunning = false
}
func (i *Instance) StartTestApp() {
	k8s.RunKubectl(i.w.t, i.w.k8sOptions, "scale", "deploy", i.w.state.testApp.name, "--replicas=1")
	AssertGslbStatus(i.w.t, i.w.k8sOptions, i.w.state.gslb.name, i.w.state.gslb.host+":Healthy")
	i.w.state.testApp.isRunning = true
}
func (i *Instance) WaitForGSLB(instances ...*Instance) ([]string, error) {
	var expectedIPs []string
	instances = append(instances, i)
	for _, in := range instances {
		if in.w.state.testApp.isRunning {
			ip := in.GetLocalTargets()
			expectedIPs = append(expectedIPs, ip...)
		}
	}
	return waitForLocalGSLBNew(i.w.t, i.w.state.gslb.host, i.w.state.gslb.port, expectedIPs)
}