func ipAddrsResult(ipAddrs string, conf types.NetConf, args *skel.CmdArgs, logger *logrus.Entry) (*current.Result, error) {
	logger.Infof("Parsing annotation \"cni.projectcalico.org/ipAddrs\":%s", ipAddrs)
	ipList, err := validateAndExtractIPs(ipAddrs, "cni.projectcalico.org/ipAddrs", logger)
	if err != nil {
		return nil, err
	}
	result := current.Result{}
	for _, ip := range ipList {
		r, err := callIPAMWithIP(ip, conf, args, logger)
		if err != nil {
			return nil, fmt.Errorf("error getting IP from IPAM: %s", err)
		}
		result.IPs = append(result.IPs, r.IPs[0])
		logger.Debugf("Adding IPv%s: %s to result", r.IPs[0].Version, ip.String())
	}
	return &result, nil
}
ipAddrsResult parses the ipAddrs annotation and calls the configured IPAM plugin for
each IP passed to it by setting the IP field in CNI_ARGS, and returns the result of calling the IPAM plugin.
Example annotation value string: "[\"10.0.0.1\", \"2001:db8::1\"]"
package k8s
import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"net"
	"os"
	"strings"
	"time"
	"github.com/containernetworking/cni/pkg/skel"
	cnitypes "github.com/containernetworking/cni/pkg/types"
	"github.com/containernetworking/cni/pkg/types/current"
	"github.com/containernetworking/plugins/pkg/ipam"
	"github.com/sirupsen/logrus"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/tools/clientcmd"
	libapi "github.com/projectcalico/libcalico-go/lib/apis/v3"
	k8sconversion "github.com/projectcalico/libcalico-go/lib/backend/k8s/conversion"
	calicoclient "github.com/projectcalico/libcalico-go/lib/clientv3"
	cerrors "github.com/projectcalico/libcalico-go/lib/errors"
	cnet "github.com/projectcalico/libcalico-go/lib/net"
	"github.com/projectcalico/libcalico-go/lib/options"
	"github.com/projectcalico/cni-plugin/internal/pkg/utils"
	"github.com/projectcalico/cni-plugin/internal/pkg/utils/cri"
	"github.com/projectcalico/cni-plugin/pkg/dataplane"
	"github.com/projectcalico/cni-plugin/pkg/types"
)
func CmdAddK8s(ctx context.Context, args *skel.CmdArgs, conf types.NetConf, epIDs utils.WEPIdentifiers, calicoClient calicoclient.Interface, endpoint *libapi.WorkloadEndpoint) (*current.Result, error) {
	var err error
	var result *current.Result
	utils.ConfigureLogging(conf)
	logger := logrus.WithFields(logrus.Fields{
		"WorkloadEndpoint": epIDs.WEPName,
		"ContainerID":      epIDs.ContainerID,
		"Pod":              epIDs.Pod,
		"Namespace":        epIDs.Namespace,
	})
	d, err := dataplane.GetDataplane(conf, logger)
	if err != nil {
		return nil, err
	}
	logger.Info("Extracted identifiers for CmdAddK8s")
	result, err = utils.CheckForSpuriousDockerAdd(args, conf, epIDs, endpoint, logger)
	if result != nil || err != nil {
		return result, err
	}
	client, err := NewK8sClient(conf, logger)
	if err != nil {
		return nil, err
	}
	logger.WithField("client", client).Debug("Created Kubernetes client")
	var routes []*net.IPNet
	if conf.IPAM.Type == "host-local" {
		var stdinData map[string]interface{}
		if err := json.Unmarshal(args.StdinData, &stdinData); err != nil {
			return nil, err
		}
		var cachedPodCidr string
		getRealPodCIDR := func() (string, error) {
			if cachedPodCidr == "" {
				var err error
				cachedPodCidr, err = getPodCidr(client, conf, epIDs.Node)
				if err != nil {
					return "", err
				}
			}
			return cachedPodCidr, nil
		}
		err = utils.ReplaceHostLocalIPAMPodCIDRs(logger, stdinData, getRealPodCIDR)
		if err != nil {
			return nil, err
		}
		args.StdinData, err = json.Marshal(stdinData)
		if err != nil {
			return nil, err
		}
		logger.Debug("Updated stdin data")
		ipamData := stdinData["ipam"].(map[string]interface{})
		untypedRoutes := ipamData["routes"]
		hlRoutes, ok := untypedRoutes.([]interface{})
		if untypedRoutes != nil && !ok {
			return nil, fmt.Errorf(
				"failed to parse host-local IPAM routes section; expecting list, not: %v", stdinData["ipam"])
		}
		for _, route := range hlRoutes {
			route := route.(map[string]interface{})
			untypedDst, ok := route["dst"]
			if !ok {
				logger.Debug("Ignoring host-ipam route with no dst")
				continue
			}
			dst, ok := untypedDst.(string)
			if !ok {
				return nil, fmt.Errorf(
					"invalid IPAM routes section; expecting 'dst' to be a string, not: %v", untypedDst)
			}
			_, cidr, err := net.ParseCIDR(dst)
			if err != nil {
				logger.WithError(err).WithField("routeDest", dst).Error(
					"Failed to parse destination of host-local IPAM route in CNI configuration.")
				return nil, err
			}
			routes = append(routes, cidr)
		}
	}
	if len(routes) == 0 {
		logger.Debug("No routes specified in CNI configuration, using defaults.")
		routes = utils.DefaultRoutes
	} else {
		if conf.IncludeDefaultRoutes {
			logger.Debug("Including Calico default routes in addition to routes from CNI config")
			routes = append(utils.DefaultRoutes, routes...)
		}
		logger.WithField("routes", routes).Info("Using custom routes from CNI configuration.")
	}
	labels := make(map[string]string)
	annot := make(map[string]string)
	var ports []libapi.WorkloadEndpointPort
	var profiles []string
	var generateName string
	var serviceAccount string
	if conf.Policy.PolicyType == "k8s" {
		annotNS, err := getK8sNSInfo(client, epIDs.Namespace)
		if err != nil {
			return nil, err
		}
		logger.WithField("NS Annotations", annotNS).Debug("Fetched K8s namespace annotations")
		labels, annot, ports, profiles, generateName, serviceAccount, err = getK8sPodInfo(client, epIDs.Pod, epIDs.Namespace)
		if err != nil {
			return nil, err
		}
		logger.WithField("labels", labels).Debug("Fetched K8s labels")
		logger.WithField("annotations", annot).Debug("Fetched K8s annotations")
		logger.WithField("ports", ports).Debug("Fetched K8s ports")
		logger.WithField("profiles", profiles).Debug("Generated profiles")
		if conf.IPAM.Type == "calico-ipam" {
			var v4pools, v6pools string
			v4pools = annotNS["cni.projectcalico.org/ipv4pools"]
			v6pools = annotNS["cni.projectcalico.org/ipv6pools"]
			v4poolpod := annot["cni.projectcalico.org/ipv4pools"]
			if len(v4poolpod) != 0 {
				v4pools = v4poolpod
			}
			v6poolpod := annot["cni.projectcalico.org/ipv6pools"]
			if len(v6poolpod) != 0 {
				v6pools = v6poolpod
			}
			if len(v4pools) != 0 || len(v6pools) != 0 {
				var stdinData map[string]interface{}
				if err := json.Unmarshal(args.StdinData, &stdinData); err != nil {
					return nil, err
				}
				var v4PoolSlice, v6PoolSlice []string
				if len(v4pools) > 0 {
					if err := json.Unmarshal([]byte(v4pools), &v4PoolSlice); err != nil {
						logger.WithField("IPv4Pool", v4pools).Error("Error parsing IPv4 IPPools")
						return nil, err
					}
					if _, ok := stdinData["ipam"].(map[string]interface{}); !ok {
						return nil, errors.New("data on stdin was of unexpected type")
					}
					stdinData["ipam"].(map[string]interface{})["ipv4_pools"] = v4PoolSlice
					logger.WithField("ipv4_pools", v4pools).Debug("Setting IPv4 Pools")
				}
				if len(v6pools) > 0 {
					if err := json.Unmarshal([]byte(v6pools), &v6PoolSlice); err != nil {
						logger.WithField("IPv6Pool", v6pools).Error("Error parsing IPv6 IPPools")
						return nil, err
					}
					if _, ok := stdinData["ipam"].(map[string]interface{}); !ok {
						return nil, errors.New("data on stdin was of unexpected type")
					}
					stdinData["ipam"].(map[string]interface{})["ipv6_pools"] = v6PoolSlice
					logger.WithField("ipv6_pools", v6pools).Debug("Setting IPv6 Pools")
				}
				newData, err := json.Marshal(stdinData)
				if err != nil {
					logger.WithField("stdinData", stdinData).Error("Error Marshaling data")
					return nil, err
				}
				args.StdinData = newData
				logger.Debug("Updated stdin data")
			}
		}
	}
	ipAddrsNoIpam := annot["cni.projectcalico.org/ipAddrsNoIpam"]
	ipAddrs := annot["cni.projectcalico.org/ipAddrs"]
	switch {
	case ipAddrs == "" && ipAddrsNoIpam == "":
		result, err = utils.AddIPAM(conf, args, logger)
		if err != nil {
			return nil, err
		}
	case ipAddrs != "" && ipAddrsNoIpam != "":
		e := fmt.Errorf("can't have both annotations: 'ipAddrs' and 'ipAddrsNoIpam' in use at the same time")
		logger.Error(e)
		return nil, e
	case ipAddrsNoIpam != "":
		if conf.IPAM.Type != "calico-ipam" {
			e := fmt.Errorf("ipAddrsNoIpam is not compatible with configured IPAM: %s", conf.IPAM.Type)
			logger.Error(e)
			return nil, e
		}
		if !conf.FeatureControl.IPAddrsNoIpam {
			e := fmt.Errorf("requested feature is not enabled: ip_addrs_no_ipam")
			logger.Error(e)
			return nil, e
		}
		overriddenResult, err := overrideIPAMResult(ipAddrsNoIpam, logger)
		if err != nil {
			return nil, err
		}
		logger.Debugf("Bypassing IPAM to set the result to: %+v", overriddenResult)
		result, err = current.NewResultFromResult(overriddenResult)
		if err != nil {
			return nil, err
		}
		if len(result.IPs) == 0 {
			return nil, errors.New("failed to build result")
		}
	case ipAddrs != "":
		if conf.IPAM.Type != "calico-ipam" {
			e := fmt.Errorf("ipAddrs is not compatible with configured IPAM: %s", conf.IPAM.Type)
			logger.Error(e)
			return nil, e
		}
		if endpoint != nil {
			logger.Info("Endpoint already exists and ipAddrs is set. Release any old IPs")
			if err := releaseIPAddrs(endpoint.Spec.IPNetworks, calicoClient, logger); err != nil {
				return nil, fmt.Errorf("failed to release ipAddrs: %s", err)
			}
		}
		result, err = ipAddrsResult(ipAddrs, conf, args, logger)
		if err != nil {
			return nil, err
		}
		logger.Debugf("IPAM result set to: %+v", result)
	}
	if endpoint == nil {
		logger.Debug("Initializing new WorkloadEndpoint resource")
		endpoint = libapi.NewWorkloadEndpoint()
	}
	endpoint.Name = epIDs.WEPName
	endpoint.Namespace = epIDs.Namespace
	endpoint.Labels = labels
	endpoint.GenerateName = generateName
	endpoint.Spec.Endpoint = epIDs.Endpoint
	endpoint.Spec.Node = epIDs.Node
	endpoint.Spec.Orchestrator = epIDs.Orchestrator
	endpoint.Spec.Pod = epIDs.Pod
	endpoint.Spec.Ports = ports
	endpoint.Spec.IPNetworks = []string{}
	endpoint.Spec.ServiceAccountName = serviceAccount
	if conf.Policy.PolicyType == "k8s" {
		endpoint.Spec.Profiles = profiles
	} else {
		endpoint.Spec.Profiles = []string{conf.Name}
	}
	if err = utils.PopulateEndpointNets(endpoint, result); err != nil {
		utils.ReleaseIPAllocation(logger, conf, args)
		return nil, err
	}
	logger.WithField("endpoint", endpoint).Info("Populated endpoint")
	logger.Infof("Calico CNI using IPs: %s", endpoint.Spec.IPNetworks)
	releaseIPAM := func() {
		logger.WithField("endpointIPs", endpoint.Spec.IPNetworks).Info("Releasing IPAM allocation(s) after failure")
		utils.ReleaseIPAllocation(logger, conf, args)
	}
	desiredVethName := k8sconversion.NewConverter().VethNameForWorkload(epIDs.Namespace, epIDs.Pod)
	hostVethName, contVethMac, err := d.DoNetworking(
		ctx, calicoClient, args, result, desiredVethName, routes, endpoint, annot)
	if err != nil {
		logger.WithError(err).Error("Error setting up networking")
		releaseIPAM()
		return nil, err
	}
	mac, err := net.ParseMAC(contVethMac)
	if err != nil {
		logger.WithError(err).WithField("mac", mac).Error("Error parsing container MAC")
		releaseIPAM()
		return nil, err
	}
	endpoint.Spec.MAC = mac.String()
	endpoint.Spec.InterfaceName = hostVethName
	endpoint.Spec.ContainerID = epIDs.ContainerID
	logger.WithField("endpoint", endpoint).Info("Added Mac, interface name, and active container ID to endpoint")
	if conf.Mode == "vxlan" {
		_, subNet, _ := net.ParseCIDR(result.IPs[0].Address.String())
		var err error
		for attempts := 3; attempts > 0; attempts-- {
			err = utils.EnsureVXLANTunnelAddr(ctx, calicoClient, epIDs.Node, subNet, conf.Name)
			if err != nil {
				logger.WithError(err).Warn("Failed to set node's VXLAN tunnel IP, node may not receive traffic.  May retry...")
				time.Sleep(1 * time.Second)
				continue
			}
			break
		}
		if err != nil {
			logger.WithError(err).Error("Failed to set node's VXLAN tunnel IP after retries, node may not receive traffic.")
		}
	}
	floatingIPs := annot["cni.projectcalico.org/floatingIPs"]
	if floatingIPs != "" {
		if !conf.FeatureControl.FloatingIPs {
			releaseIPAM()
			return nil, fmt.Errorf("requested feature is not enabled: floating_ips")
		}
		ips, err := parseIPAddrs(floatingIPs, logger)
		if err != nil {
			releaseIPAM()
			return nil, err
		}
		var podnetV4, podnetV6 net.IPNet
		for _, ipNet := range result.IPs {
			if ipNet.Address.IP.To4() != nil {
				podnetV4 = ipNet.Address
				netmask, _ := podnetV4.Mask.Size()
				if netmask != 32 {
					return nil, fmt.Errorf("PodIP %v is not a valid IPv4: Mask size is %d, not 32", ipNet, netmask)
				}
			} else {
				podnetV6 = ipNet.Address
				netmask, _ := podnetV6.Mask.Size()
				if netmask != 128 {
					return nil, fmt.Errorf("PodIP %v is not a valid IPv6: Mask size is %d, not 128", ipNet, netmask)
				}
			}
		}
		for _, ip := range ips {
			if strings.Contains(ip, ":") {
				endpoint.Spec.IPNATs = append(endpoint.Spec.IPNATs, libapi.IPNAT{
					InternalIP: podnetV6.IP.String(),
					ExternalIP: ip,
				})
			} else {
				endpoint.Spec.IPNATs = append(endpoint.Spec.IPNATs, libapi.IPNAT{
					InternalIP: podnetV4.IP.String(),
					ExternalIP: ip,
				})
			}
		}
		logger.WithField("endpoint", endpoint).Info("Added floatingIPs to endpoint")
	}
	if _, err := utils.CreateOrUpdate(ctx, calicoClient, endpoint); err != nil {
		logger.WithError(err).Error("Error creating/updating endpoint in datastore.")
		releaseIPAM()
		return nil, err
	}
	logger.Info("Wrote updated endpoint to datastore")
	result.Interfaces = append(result.Interfaces, &current.Interface{
		Name: endpoint.Spec.InterfaceName},
	)
	return result, nil
}
func CmdDelK8s(ctx context.Context, c calicoclient.Interface, epIDs utils.WEPIdentifiers, args *skel.CmdArgs, conf types.NetConf, logger *logrus.Entry) error {
	d, err := dataplane.GetDataplane(conf, logger)
	if err != nil {
		return err
	}
	if cri.IsDockershimV1(args.Netns) {
		err = utils.RegisterDeletedWep(args.ContainerID)
		if err != nil {
			logger.WithError(err).Warn("Failed to register pod deletion timestamp.")
			return err
		}
	}
	for attempts := 5; attempts >= 0; attempts-- {
		wep, err := c.WorkloadEndpoints().Get(ctx, epIDs.Namespace, epIDs.WEPName, options.GetOptions{})
		if err != nil {
			if _, ok := err.(cerrors.ErrorResourceDoesNotExist); !ok {
				return err
			}
			logger.WithField("WorkloadEndpoint", epIDs.WEPName).Warning("WorkloadEndpoint does not exist in the datastore, moving forward with the clean up")
		} else if wep.Spec.ContainerID != "" && args.ContainerID != wep.Spec.ContainerID {
			logger.WithField("WorkloadEndpoint", wep).Warning("CNI_CONTAINERID does not match WorkloadEndpoint ConainerID, don't delete WEP.")
		} else if _, err = c.WorkloadEndpoints().Delete(
			ctx,
			wep.Namespace,
			wep.Name,
			options.DeleteOptions{
				ResourceVersion: wep.ResourceVersion,
				UID:             &wep.UID,
			},
		); err != nil {
			switch err := err.(type) {
			case cerrors.ErrorResourceDoesNotExist:
				logger.WithField("endpoint", wep).Info("Endpoint object does not exist, no need to clean up.")
			case cerrors.ErrorResourceUpdateConflict:
				if attempts == 0 {
					logger.WithField("endpoint", wep).Warn("Endpoint was modified before it could be deleted.  Giving up.")
					return fmt.Errorf("error deleting endpoint: endpoint was modified before it could be deleted: %v", err)
				}
				logger.WithField("endpoint", wep).Info("Endpoint was modified before it could be deleted.  Retrying...")
				continue
			case cerrors.ErrorOperationNotSupported:
				logger.WithField("endpoint", wep).WithError(err).Warn("Deleting pod returned ErrorOperationNotSupported.")
			default:
				return err
			}
		}
		break
	}
	logger.Info("Cleaning up netns")
	err = d.CleanUpNamespace(args)
	if err != nil {
		return err
	}
	logger.Info("Releasing IP address(es)")
	err = utils.DeleteIPAM(conf, args, logger)
	if err != nil {
		return err
	}
	logger.Info("Teardown processing complete.")
	return nil
}
func releaseIPAddrs(ipAddrs []string, calico calicoclient.Interface, logger *logrus.Entry) error {
	for _, ip := range ipAddrs {
		log := logger.WithField("IP", ip)
		log.Info("Releasing explicitly requested address")
		cip, _, err := cnet.ParseCIDR(ip)
		if err != nil {
			return err
		}
		unallocated, err := calico.IPAM().ReleaseIPs(context.Background(), []cnet.IP{*cip})
		if err != nil {
			log.WithError(err).Error("Failed to release explicit IP")
			return err
		}
		if len(unallocated) > 0 {
			log.Warn("Asked to release address but it doesn't exist.")
		} else {
			log.Infof("Released explicit address: %s", ip)
		}
	}
	return nil
}