func DeleteContainerWithId(netconf, podName, podNamespace, containerId, k8sNs string) (exitCode int, err error) {
	return DeleteContainerWithIdAndIfaceName(netconf, podName, podNamespace, containerId, "eth0", k8sNs)
}
func DeleteContainerWithId(netconf, netnspath, podName, podNamespace, containerId string) (exitCode int, err error) {
package testutils
import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"math/rand"
	"net"
	"os"
	"os/exec"
	"strings"
	"time"
	"github.com/projectcalico/cni-plugin/pkg/dataplane/windows"
	"github.com/Microsoft/hcsshim"
	"github.com/containernetworking/cni/pkg/invoke"
	"github.com/containernetworking/cni/pkg/skel"
	"github.com/containernetworking/cni/pkg/types"
	types020 "github.com/containernetworking/cni/pkg/types/020"
	"github.com/containernetworking/cni/pkg/types/current"
	"github.com/mcuadros/go-version"
	"github.com/projectcalico/cni-plugin/pkg/k8s"
	plugintypes "github.com/projectcalico/cni-plugin/pkg/types"
	client "github.com/projectcalico/libcalico-go/lib/clientv3"
	log "github.com/sirupsen/logrus"
	kerrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"golang.org/x/sys/windows/registry"
)
const HnsNoneNs = "none"
func WipeK8sPods(netconf string) {
	conf := plugintypes.NetConf{}
	if err := json.Unmarshal([]byte(netconf), &conf); err != nil {
		panic(err)
	}
	logger := log.WithFields(log.Fields{
		"Namespace": HnsNoneNs,
	})
	clientset, err := k8s.NewK8sClient(conf, logger)
	if err != nil {
		panic(err)
	}
	log.WithField("clientset:", clientset).Info("DEBUG")
	pods, err := clientset.CoreV1().Pods(K8S_TEST_NS).List(context.Background(), metav1.ListOptions{})
	if err != nil {
		panic(err)
	}
	for _, pod := range pods.Items {
		err = clientset.CoreV1().Pods(K8S_TEST_NS).Delete(context.Background(), pod.Name, metav1.DeleteOptions{})
		if err != nil {
			if kerrors.IsNotFound(err) {
				continue
			}
			panic(err)
		}
	}
	log.Info("WipeK8sPods Sucess")
}
func CreateContainerUsingDocker() (string, error) {
	var image string
	if os.Getenv("WINDOWS_OS") == "Windows1903container" {
		image = "mcr.microsoft.com/windows/servercore/insider:10.0.18317.1000"
	} else if os.Getenv("WINDOWS_OS") == "Windows1809container" {
		image = "mcr.microsoft.com/windows/servercore:1809"
	}
	command := fmt.Sprintf("docker run --net none -d %s powershell.exe -command start-sleep -s 300", image)
	cmd := exec.Command("powershell.exe", command)
	out, err := cmd.CombinedOutput()
	if err != nil {
		log.Warn(err)
		return "", err
	}
	temp := strings.TrimSpace(string(out))
	log.Debugf("container ID: %s", temp)
	return temp, nil
}
func CreateContainerByIdUsingContainerd(overrideContainerID string) (string, string, error) {
	return createContainerUsingContainerd(overrideContainerID)
}
func CreateContainerUsingContainerd() (string, string, error) {
	containerId := fmt.Sprintf("ctr%d", rand.Uint32())
	return createContainerUsingContainerd(containerId)
}
func GetContainerNamespace(containerId string) (string, error) {
	command := fmt.Sprintf("Get-HnsNamespace | Where Containers -eq %v | Select-Object -expandproperty ID", containerId)
	cmd := exec.Command("powershell.exe", command)
	log.Infof("Running powershell command: %v", command)
	out, err := cmd.CombinedOutput()
	if err != nil {
		log.WithError(err).Info("could not get namespace for container")
		return "", err
	}
	return strings.TrimSpace(string(out)), nil
}
func createContainerUsingContainerd(containerId string) (string, string, error) {
	image := "k8s.gcr.io/pause:3.5"
	command := fmt.Sprintf(`& 'C:\Program Files\containerd\ctr.exe' images pull %v`, image)
	cmd := exec.Command("powershell.exe", command)
	log.Infof("Running powershell command: %v", command)
	_, err := cmd.CombinedOutput()
	if err != nil {
		return "", "", errors.New(fmt.Sprintf("failed to pull image: %v", err))
	}
	command = fmt.Sprintf(`& 'C:\Program Files\containerd\ctr.exe' run --detach %v %v`, image, containerId)
	cmd = exec.Command("powershell.exe", command)
	log.Infof("Running powershell command: %v", command)
	_, err = cmd.CombinedOutput()
	if err != nil {
		return "", "", errors.New(fmt.Sprintf("failed to create container: %v", err))
	}
	ns, err := GetContainerNamespace(containerId)
	if err != nil {
		return "", "", errors.New(fmt.Sprintf("failed to get container namespace: %v", err))
	}
	log.Infof("ctr namespace: %s", ns)
	return containerId, ns, nil
}
func DeleteRunningContainer(containerId string) error {
	if os.Getenv("CONTAINER_RUNTIME") != "containerd" {
		command := fmt.Sprintf("docker rm -f %s", containerId)
		cmd := exec.Command("powershell.exe", command)
		_, err := cmd.CombinedOutput()
		if err != nil {
			log.WithError(err).WithField("id", containerId).Error("Failed to stop docker container")
			return err
		}
		return nil
	}
	cmds := []string{
		fmt.Sprintf(`& 'C:\Program Files\containerd\ctr.exe' tasks kill %v`, containerId),
		fmt.Sprintf(`& 'C:\Program Files\containerd\ctr.exe' containers delete %v`, containerId),
	}
	for _, c := range cmds {
		cmd := exec.Command("powershell.exe", c)
		log.Infof("Running powershell command: %v", c)
		_, err := cmd.CombinedOutput()
		if err != nil {
			log.WithError(err).WithField("id", containerId).Warn("Failed to stop docker container")
			return err
		}
	}
	return nil
}
func CreateContainer(netconf, podName, podNamespace, ip, k8sNs string) (containerID string, result *current.Result, contVeth string, contAddr []string, contRoutes []string, err error) {
	if os.Getenv("CONTAINER_RUNTIME") != "containerd" {
		containerID, err = CreateContainerUsingDocker()
		if err != nil {
			return "", nil, "", []string{}, []string{}, err
		}
		result, contVeth, contAddr, contRoutes, err = RunCNIPluginWithId(netconf, podName, podNamespace, ip, containerID, "", k8sNs)
		if err != nil {
			return containerID, nil, "", []string{}, []string{}, err
		}
		return
	}
	containerID, targetNs, err := CreateContainerUsingContainerd()
	if err != nil {
		return "", nil, "", []string{}, []string{}, err
	}
	result, contVeth, contAddr, contRoutes, err = RunCNIPluginWithId(netconf, podName, targetNs, ip, containerID, "", k8sNs)
	if err != nil {
		return containerID, nil, "", []string{}, []string{}, err
	}
	return
}
func CreateContainerWithId(netconf, podName, podNamespace, ip, overrideContainerID, k8sNs string) (containerID string, result *current.Result, contVeth string, contAddr []string, contRoutes []string, err error) {
	if os.Getenv("CONTAINER_RUNTIME") != "containerd" {
		containerID, err = CreateContainerUsingDocker()
		if err != nil {
			return "", nil, "", []string{}, []string{}, err
		}
		if overrideContainerID != "" {
			containerID = overrideContainerID
		}
		result, contVeth, contAddr, contRoutes, err = RunCNIPluginWithId(netconf, podName, podNamespace, ip, containerID, "", k8sNs)
		if err != nil {
			log.Errorf("Error: ", err)
			return containerID, nil, "", []string{}, []string{}, err
		}
	}
	containerID, targetNs, err := CreateContainerByIdUsingContainerd(overrideContainerID)
	if err != nil {
		return "", nil, "", []string{}, []string{}, err
	}
	result, contVeth, contAddr, contRoutes, err = RunCNIPluginWithId(netconf, podName, podNamespace, ip, containerID, "", targetNs)
	if err != nil {
		log.Errorf("Error: ", err)
		return containerID, nil, "", []string{}, []string{}, err
	}
	return
}
func RunCNIPluginWithId(
	netconf,
	podName,
	podNamespace,
	ip,
	containerId,
	ifName,
	k8sNs string,
) (
	result *current.Result,
	contVeth string,
	contAddr []string,
	contRoutes []string,
	err error,
) {
	k8sEnv := ""
	if podName != "" {
		k8sEnv = fmt.Sprintf("CNI_ARGS=K8S_POD_NAME=%s;K8S_POD_NAMESPACE=%s;K8S_POD_INFRA_CONTAINER_ID=whatever", podName, k8sNs)
		if ip != "" {
			k8sEnv = fmt.Sprintf("%s;IP=%s", k8sEnv, ip)
		}
	}
	if ifName == "" {
		ifName = "eth0"
	}
	env := os.Environ()
	env = append(env, []string{
		"CNI_COMMAND=ADD",
		fmt.Sprintf("CNI_IFNAME=%s", ifName),
		fmt.Sprintf("CNI_PATH=%s", os.Getenv("BIN")),
		fmt.Sprintf("CNI_CONTAINERID=%s", containerId),
		fmt.Sprintf("CNI_NETNS=%s", podNamespace),
		k8sEnv,
	}...)
	args := &cniArgs{env}
	var r types.Result
	pluginPath := fmt.Sprintf("%s\\%s", os.Getenv("BIN"), os.Getenv("PLUGIN"))
	log.Debugf("pluginPath: %v", pluginPath)
	r, err = invoke.ExecPluginWithResult(context.Background(), pluginPath, []byte(netconf), args, nil)
	if err != nil {
		log.Errorf("error from invoke.ExecPluginWithResult %v", err)
		_ = DeleteRunningContainer(containerId)
		return
	}
	var nc types.NetConf
	if err = json.Unmarshal([]byte(netconf), &nc); err != nil {
		log.Errorf("unmarshal err: ", err)
		panic(err)
	}
	if version.Compare(nc.CNIVersion, "0.3.0", "<") {
		var out []byte
		out, err = json.Marshal(r)
		log.Infof("CNI output: %s", out)
		r020 := types020.Result{}
		if err = json.Unmarshal(out, &r020); err != nil {
			log.Errorf("Error unmarshaling output to Result: %v", err)
			return
		}
		result, err = current.NewResultFromResult(&r020)
		if err != nil {
			return
		}
	} else {
		result, err = current.GetResult(r)
		if err != nil {
			return
		}
	}
	return
}
func DeleteContainer(netconf, podName, podNamespace, k8sNs string) (exitCode int, err error) {
	return DeleteContainerWithId(netconf, podName, podNamespace, "", k8sNs)
}