func (c *TestContext) postProvision(gravityNodes []*gravity) error {
	ctx, cancel := context.WithTimeout(c.Context(), clockSyncTimeout)
	defer cancel()
	c.Logger().Debug("synchronizing clocks")
	var timeNodes []sshutil.SshNode
	for _, node := range gravityNodes {
		timeNodes = append(timeNodes, sshutil.SshNode{Client: node.Client(), Log: node.Logger()})
	}
	if err := sshutil.WaitTimeSync(ctx, timeNodes); err != nil {
		return trace.Wrap(err)
	}
	return nil
}
postProvision runs common tasks for both ops and cloud provisioners once the VMs have been setup and are running
package gravity
import (
	"context"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"sort"
	"strings"
	"time"
	"github.com/gravitational/robotest/infra"
	"github.com/gravitational/robotest/infra/terraform"
	"github.com/gravitational/robotest/lib/constants"
	sshutil "github.com/gravitational/robotest/lib/ssh"
	"github.com/gravitational/robotest/lib/utils"
	"github.com/gravitational/robotest/lib/wait"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/dustin/go-humanize"
	"github.com/gravitational/trace"
	uuid "github.com/satori/go.uuid"
	"github.com/sirupsen/logrus"
)
type cloudDynamicParams struct {
	ProvisionerConfig
	user      string
	homeDir   string
	terraform terraform.Config
	env       map[string]string
}
func configureVMs(baseCtx context.Context, log logrus.FieldLogger, params cloudDynamicParams, nodes []*gravity) error {
	errChan := make(chan error, len(nodes))
	ctx, cancel := context.WithCancel(baseCtx)
	defer cancel()
	for _, node := range nodes {
		go func(node *gravity) {
			err := configureVM(ctx, log, node, params)
			errChan <- err
		}(node)
	}
	err := utils.CollectErrors(ctx, errChan)
	if err != nil {
		return trace.Wrap(err)
	}
	return nil
}
func (c *TestContext) Provision(cfg ProvisionerConfig) (cluster Cluster, err error) {
	c.provisionerCfg = cfg
	switch cfg.CloudProvider {
	case constants.Azure, constants.AWS, constants.GCE:
		var config *terraform.Config
		cluster, config, err = c.provisionCloud(cfg)
		if err == nil && cfg.CloudProvider == constants.GCE {
			c.WithFields(logrus.Fields{"node_tag": config.GCE.NodeTag})
		}
	case constants.Ops:
		cluster, err = c.provisionOps(cfg)
	default:
		err = trace.BadParameter("unkown cloud provider: %q", cfg.CloudProvider)
	}
	if err != nil && cluster.Destroy != nil {
		destroyErr := cluster.Destroy()
		if destroyErr != nil {
			err = trace.NewAggregate(err, destroyErr)
		}
		cluster.Nodes = nil
		cluster.Destroy = nil
	}
	return cluster, trace.Wrap(err)
}
func (c *TestContext) provisionOps(cfg ProvisionerConfig) (cluster Cluster, err error) {
	c.Logger().WithField("config", cfg).Debug("Provisioning via Ops Center")
	c.Logger().Debug("attempting to connect to AWS api")
	sess, err := session.NewSession(&aws.Config{
		Region:      aws.String(cfg.Ops.EC2Region),
		Credentials: credentials.NewStaticCredentials(cfg.Ops.EC2AccessKey, cfg.Ops.EC2SecretKey, ""),
	})
	if err != nil {
		return cluster, trace.Wrap(err)
	}
	c.Logger().Debug("logging into the ops center")
	out, err := exec.Command("tele", "login", "-o", cfg.Ops.URL, "--key", cfg.Ops.OpsKey).CombinedOutput()
	if err != nil {
		return cluster, trace.WrapWithMessage(err, string(out))
	}
	clusterName := fmt.Sprint(c.name, "-", uuid.NewV4().String())
	c.provisionerCfg.clusterName = clusterName
	c.Logger().Debug("Generated cluster name: ", clusterName)
	c.Logger().Debug("validating configuration")
	err = validateConfig(cfg)
	if err != nil {
		return cluster, trace.Wrap(err)
	}
	c.Logger().Debug("generating ops center cluster configuration")
	clusterPath := path.Join(cfg.StateDir, "cluster.yaml")
	err = os.MkdirAll(cfg.StateDir, constants.SharedDirMask)
	if err != nil {
		return cluster, trace.Wrap(err)
	}
	defn, err := generateClusterConfig(cfg, clusterName)
	if err != nil {
		return cluster, trace.Wrap(err)
	}
	err = ioutil.WriteFile(clusterPath, []byte(defn), constants.SharedReadMask)
	if err != nil {
		return cluster, trace.Wrap(err)
	}
	c.Logger().Debug("requesting ops center to provision our cluster")
	out, err = exec.Command("tele", "create", clusterPath).CombinedOutput()
	if err != nil {
		return cluster, trace.Wrap(err, string(out))
	}
	cluster.Destroy = cfg.DestroyOpsFn(c, clusterName)
	timeout := time.After(cloudInitTimeout)
	ticker := time.NewTicker(15 * time.Second)
	defer ticker.Stop()
	c.Logger().Debug("waiting for ops center provisioning to complete")
Loop:
	for {
		select {
		case <-timeout:
			return cluster, errors.New("clusterInitTimeout exceeded")
		case <-ticker.C:
			status, err := getTeleClusterStatus(clusterName)
			c.Logger().WithField("status", status).Debug("provisioning status")
			if err != nil {
				return cluster, trace.Wrap(err)
			}
			switch status {
			case "installing":
			case "active":
				break Loop
			default:
				return cluster, trace.BadParameter("unexpected cluster status: %v", status)
			}
		}
	}
	c.Logger().Debug("Attempting to get a listing of instances from AWS for the cluster.")
	ec2svc := ec2.New(sess)
	gravityNodes, err := c.getAWSNodes(ec2svc, "tag:KubernetesCluster", clusterName)
	if err != nil {
		return cluster, trace.Wrap(err)
	}
	c.Logger().Debugf("Running post provisioning tasks.")
	err = c.postProvision(gravityNodes)
	if err != nil {
		return cluster, trace.Wrap(err)
	}
	err = validateDiskSpeed(c.Context(), gravityNodes, cfg.dockerDevice, c.Logger())
	if err != nil {
		return cluster, trace.Wrap(err)
	}
	cluster.Nodes = asNodes(gravityNodes)
	c.Logger().Info("Provisioning complete.")
	return cluster, nil
}
func (c *TestContext) provisionCloud(cfg ProvisionerConfig) (cluster Cluster, config *terraform.Config, err error) {
	log := c.Logger().WithField("config", cfg)
	log.Debug("Provisioning VMs.")
	err = validateConfig(cfg)
	if err != nil {
		return cluster, nil, trace.Wrap(err)
	}
	infra, err := runTerraform(c.Context(), cfg, c.Logger())
	if err != nil {
		return cluster, nil, trace.Wrap(err)
	}
	defer func() {
		if err == nil || infra.destroyFn == nil {
			return
		}
		if errDestroy := destroyResource(infra.destroyFn); errDestroy != nil {
			log.WithError(errDestroy).Error("Failed to destroy resources.")
		}
	}()
	ctx, cancel := context.WithTimeout(c.Context(), cloudInitTimeout)
	defer cancel()
	log.Debug("Connecting to VMs.")
	gravityNodes, err := connectVMs(ctx, c.Logger(), infra.params, infra.nodes)
	if err != nil {
		log.WithError(err).Error("Some nodes failed to connect, tear down as unusable.")
		return cluster, nil, trace.NewAggregate(err, destroyResource(infra.destroyFn))
	}
	c.streamLogs(gravityNodes)
	log.Debug("Configuring VMs.")
	err = configureVMs(ctx, c.Logger(), infra.params, gravityNodes)
	if err != nil {
		log.WithError(err).Error("Some nodes failed to initialize, tear down as non-usable.")
		return cluster, nil, trace.NewAggregate(err, destroyResource(infra.destroyFn))
	}
	err = c.postProvision(gravityNodes)
	if err != nil {
		log.WithError(err).Error("Post-provisioning failed, tear down as non-usable.")
		return cluster, nil, trace.Wrap(err)
	}
	if cfg.CloudProvider == constants.Azure {
		err = validateDiskSpeed(c.Context(), gravityNodes, cfg.dockerDevice, c.Logger())
		if err != nil {
			return cluster, nil, trace.Wrap(err)
		}
	}
	log.WithField("nodes", gravityNodes).Debug("Provisioning complete.")
	nodes := asNodes(gravityNodes)
	cluster.Nodes = nodes
	cluster.Destroy = wrapDestroyFunc(c, cfg.Tag(), nodes, infra.destroyFn)
	return cluster, &infra.params.terraform, nil
}
func (c *TestContext) streamLogs(gravityNodes []*gravity) {
	c.Logger().Debug("Streaming logs.")
	for _, node := range gravityNodes {
		go func(node *gravity) {
			err := node.streamStartupLogs(c.monitorCtx)
			if err != nil && !utils.IsContextCancelledError(err) {
				c.Logger().Warnf("Failed to stream startup script logs: %v.", err)
			}
		}(node)
		go func(node *gravity) {
			if err := node.streamLogs(c.monitorCtx); err != nil {
				switch {
				case sshutil.IsExitMissingError(err):
					if c.Context().Err() != nil {
						return
					}
					c.markPreempted(node)
				case utils.IsContextCancelledError(err):
				default:
					c.Logger().Warnf("Failed to stream logs: %v.", err)
				}
			}
		}(node)
	}
}