func (os *linuxNetwork) TeardownPodENINetwork(vlanID int, log logger.Logger) error {
	log.Infof("Tear down of pod ENI namespace")
	if vlan, err := os.netLink.LinkByName(fmt.Sprintf("vlan.eth.%d",
		vlanID)); err == nil {
		err := os.netLink.LinkDel(vlan)
		if err != nil {
			return errors.Wrapf(err, "TeardownPodENINetwork: failed to delete vlan link for %d", vlanID)
		}
	}
	vlanRule := os.netLink.NewRule()
	vlanRule.Table = vlanID + 100
	vlanRule.Priority = vlanRulePriority
	for {
		if err := os.netLink.RuleDel(vlanRule); err != nil {
			if !containsNoSuchRule(err) {
				return errors.Wrapf(err, "TeardownPodENINetwork: failed to delete container rule for %d", vlanID)
			}
			break
		}
	}
	return nil
}
TeardownPodENINetwork tears down the vlan and corresponding ip rules.
package driver
import (
	"fmt"
	"net"
	"os"
	"syscall"
	"time"
	"github.com/pkg/errors"
	"golang.org/x/sys/unix"
	"github.com/containernetworking/plugins/pkg/ns"
	"github.com/vishvananda/netlink"
	"github.com/aws/amazon-vpc-cni-k8s/pkg/ipwrapper"
	"github.com/aws/amazon-vpc-cni-k8s/pkg/netlinkwrapper"
	"github.com/aws/amazon-vpc-cni-k8s/pkg/networkutils"
	"github.com/aws/amazon-vpc-cni-k8s/pkg/nswrapper"
	"github.com/aws/amazon-vpc-cni-k8s/pkg/procsyswrapper"
	"github.com/aws/amazon-vpc-cni-k8s/pkg/utils/logger"
)
const (
	vlanRulePriority = 10
	toContainerRulePriority = 512
	fromContainerRulePriority = 1536
	mainRouteTable = unix.RT_TABLE_MAIN
	WAIT_INTERVAL = 50 * time.Millisecond
	v6DADTimeout = 10 * time.Second
)
type NetworkAPIs interface {
	SetupNS(hostVethName string, contVethName string, netnsPath string, v4Addr *net.IPNet, v6Addr *net.IPNet, deviceNumber int, vpcCIDRs []string, useExternalSNAT bool, mtu int, log logger.Logger) error
	TeardownNS(addr *net.IPNet, deviceNumber int, log logger.Logger) error
	SetupPodENINetwork(hostVethName string, contVethName string, netnsPath string, v4Addr *net.IPNet, v6Addr *net.IPNet, vlanID int, eniMAC string,
		subnetGW string, parentIfIndex int, mtu int, log logger.Logger) error
	TeardownPodENINetwork(vlanID int, log logger.Logger) error
}
type linuxNetwork struct {
	netLink netlinkwrapper.NetLink
	ns      nswrapper.NS
	procSys procsyswrapper.ProcSys
}
func New() NetworkAPIs {
	return &linuxNetwork{
		netLink: netlinkwrapper.NewNetLink(),
		ns:      nswrapper.NewNS(),
		procSys: procsyswrapper.NewProcSys(),
	}
}
type createVethPairContext struct {
	contVethName string
	hostVethName string
	v4Addr       *net.IPNet
	v6Addr       *net.IPNet
	netLink      netlinkwrapper.NetLink
	ip           ipwrapper.IP
	mtu          int
	procSys      procsyswrapper.ProcSys
}
func newCreateVethPairContext(contVethName string, hostVethName string, v4Addr *net.IPNet, v6Addr *net.IPNet, mtu int) *createVethPairContext {
	return &createVethPairContext{
		contVethName: contVethName,
		hostVethName: hostVethName,
		v4Addr:       v4Addr,
		v6Addr:       v6Addr,
		netLink:      netlinkwrapper.NewNetLink(),
		ip:           ipwrapper.NewIP(),
		mtu:          mtu,
		procSys:      procsyswrapper.NewProcSys(),
	}
}
func (createVethContext *createVethPairContext) run(hostNS ns.NetNS) error {
	veth := &netlink.Veth{
		LinkAttrs: netlink.LinkAttrs{
			Name:  createVethContext.contVethName,
			Flags: net.FlagUp,
			MTU:   createVethContext.mtu,
		},
		PeerName: createVethContext.hostVethName,
	}
	if err := createVethContext.netLink.LinkAdd(veth); err != nil {
		return err
	}
	hostVeth, err := createVethContext.netLink.LinkByName(createVethContext.hostVethName)
	if err != nil {
		return errors.Wrapf(err, "setup NS network: failed to find link %q", createVethContext.hostVethName)
	}
	if err = createVethContext.netLink.LinkSetUp(hostVeth); err != nil {
		return errors.Wrapf(err, "setup NS network: failed to set link %q up", createVethContext.hostVethName)
	}
	contVeth, err := createVethContext.netLink.LinkByName(createVethContext.contVethName)
	if err != nil {
		return errors.Wrapf(err, "setup NS network: failed to find link %q", createVethContext.contVethName)
	}
	if err = createVethContext.netLink.LinkSetUp(contVeth); err != nil {
		return errors.Wrapf(err, "setup NS network: failed to set link %q up", createVethContext.contVethName)
	}
	if createVethContext.v6Addr != nil && createVethContext.v6Addr.IP.To16() != nil {
		if err = createVethContext.procSys.Set(fmt.Sprintf("net/ipv6/conf/%s/disable_ipv6", createVethContext.contVethName), "0"); err != nil {
			if !os.IsNotExist(err) {
				return errors.Wrapf(err, "setupVeth network: failed to enable IPv6 on container veth interface")
			}
		}
		if err = createVethContext.procSys.Set(fmt.Sprintf("net/ipv6/conf/lo/disable_ipv6"), "0"); err != nil {
			if !os.IsNotExist(err) {
				return errors.Wrapf(err, "setupVeth network: failed to enable IPv6 on container's lo interface")
			}
		}
	}
	var gw net.IP
	var maskLen int
	var addr *netlink.Addr
	var defNet *net.IPNet
	if createVethContext.v4Addr != nil {
		gw = net.IPv4(169, 254, 1, 1)
		maskLen = 32
		addr = &netlink.Addr{IPNet: createVethContext.v4Addr}
		defNet = &net.IPNet{IP: net.IPv4zero, Mask: net.CIDRMask(0, maskLen)}
	} else if createVethContext.v6Addr != nil {
		gw = net.IP{0xfe, 0x80, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}
		maskLen = 128
		addr = &netlink.Addr{IPNet: createVethContext.v6Addr}
		defNet = &net.IPNet{IP: net.IPv6zero, Mask: net.CIDRMask(0, maskLen)}
	}
	gwNet := &net.IPNet{IP: gw, Mask: net.CIDRMask(maskLen, maskLen)}
	if err = createVethContext.netLink.RouteReplace(&netlink.Route{
		LinkIndex: contVeth.Attrs().Index,
		Scope:     netlink.SCOPE_LINK,
		Dst:       gwNet}); err != nil {
		return errors.Wrap(err, "setup NS network: failed to add default gateway")
	}
	if err = createVethContext.netLink.RouteAdd(&netlink.Route{
		LinkIndex: contVeth.Attrs().Index,
		Scope:     netlink.SCOPE_UNIVERSE,
		Dst:       defNet,
		Gw:        gw,
	}); err != nil {
		return errors.Wrap(err, "setup NS network: failed to add default route")
	}
	if err = createVethContext.netLink.AddrAdd(contVeth, addr); err != nil {
		return errors.Wrapf(err, "setup NS network: failed to add IP addr to %q", createVethContext.contVethName)
	}
	neigh := &netlink.Neigh{
		LinkIndex:    contVeth.Attrs().Index,
		State:        netlink.NUD_PERMANENT,
		IP:           gwNet.IP,
		HardwareAddr: hostVeth.Attrs().HardwareAddr,
	}
	if err = createVethContext.netLink.NeighAdd(neigh); err != nil {
		return errors.Wrap(err, "setup NS network: failed to add static ARP")
	}
	if createVethContext.v6Addr != nil && createVethContext.v6Addr.IP.To16() != nil {
		if err := WaitForAddressesToBeStable(createVethContext.contVethName, v6DADTimeout); err != nil {
			return errors.Wrap(err, "setup NS network: failed while waiting for v6 addresses to be stable")
		}
	}
	if err = createVethContext.netLink.LinkSetNsFd(hostVeth, int(hostNS.Fd())); err != nil {
		return errors.Wrap(err, "setup NS network: failed to move veth to host netns")
	}
	return nil
}
func WaitForAddressesToBeStable(ifName string, timeout time.Duration) error {
	link, err := netlink.LinkByName(ifName)
	if err != nil {
		return fmt.Errorf("failed to retrieve link: %v", err)
	}
	deadline := time.Now().Add(timeout)
	for {
		addrs, err := netlink.AddrList(link, netlink.FAMILY_V6)
		if err != nil {
			return fmt.Errorf("could not list addresses: %v", err)
		}
		ok := true
		for _, addr := range addrs {
			if addr.Flags&(syscall.IFA_F_TENTATIVE|syscall.IFA_F_DADFAILED) > 0 {
				ok = false
				break
			}
		}
		if ok {
			return nil
		}
		if time.Now().After(deadline) {
			return fmt.Errorf("link %s still has tentative addresses after %d seconds",
				ifName,
				timeout)
		}
		time.Sleep(WAIT_INTERVAL)
	}
}
func (os *linuxNetwork) SetupNS(hostVethName string, contVethName string, netnsPath string, v4Addr *net.IPNet, v6Addr *net.IPNet,
	deviceNumber int, vpcCIDRs []string, useExternalSNAT bool, mtu int, log logger.Logger) error {
	log.Debugf("SetupNS: hostVethName=%s, contVethName=%s, netnsPath=%s, deviceNumber=%d, mtu=%d", hostVethName, contVethName, netnsPath, deviceNumber, mtu)
	return setupNS(hostVethName, contVethName, netnsPath, v4Addr, v6Addr, deviceNumber, vpcCIDRs, useExternalSNAT, os.netLink, os.ns, mtu, log, os.procSys)
}
func setupNS(hostVethName string, contVethName string, netnsPath string, v4Addr *net.IPNet, v6Addr *net.IPNet, deviceNumber int, vpcCIDRs []string, useExternalSNAT bool,
	netLink netlinkwrapper.NetLink, ns nswrapper.NS, mtu int, log logger.Logger, procSys procsyswrapper.ProcSys) error {
	hostVeth, err := setupVeth(hostVethName, contVethName, netnsPath, v4Addr, v6Addr, netLink, ns, mtu, procSys, log)
	if err != nil {
		return errors.Wrapf(err, "setupNS network: failed to setup veth pair.")
	}
	log.Debugf("Setup host route outgoing hostVeth, LinkIndex %d", hostVeth.Attrs().Index)
	var addrHostAddr *net.IPNet
	if v4Addr != nil && v4Addr.IP.To4() != nil {
		addrHostAddr = &net.IPNet{
			IP:   v4Addr.IP,
			Mask: net.CIDRMask(32, 32)}
	} else if v6Addr != nil && v6Addr.IP.To16() != nil {
		addrHostAddr = &net.IPNet{
			IP:   v6Addr.IP,
			Mask: net.CIDRMask(128, 128)}
	}
	route := netlink.Route{
		LinkIndex: hostVeth.Attrs().Index,
		Scope:     netlink.SCOPE_LINK,
		Dst:       addrHostAddr}
	if err := netLink.RouteReplace(&route); err != nil {
		return errors.Wrapf(err, "setupNS: unable to add or replace route entry for %s", route.Dst.IP.String())
	}
	log.Debugf("Successfully set host route to be %s/0", route.Dst.IP.String())
	err = addContainerRule(netLink, true, addrHostAddr, mainRouteTable)
	if err != nil {
		log.Errorf("Failed to add toContainer rule for %s err=%v, ", addrHostAddr.String(), err)
		return errors.Wrap(err, "setupNS network: failed to add toContainer")
	}
	log.Infof("Added toContainer rule for %s", addrHostAddr.String())
	if deviceNumber > 0 {
		tableNumber := deviceNumber + 1
		err = addContainerRule(netLink, false, addrHostAddr, tableNumber)
		if err != nil {
			log.Errorf("Failed to add fromContainer rule for %s err: %v", addrHostAddr.String(), err)
			return errors.Wrap(err, "add NS network: failed to add fromContainer rule")
		}
		log.Infof("Added rule priority %d from %s table %d", fromContainerRulePriority, addrHostAddr.String(), tableNumber)
	}
	return nil
}
func setupVeth(hostVethName string, contVethName string, netnsPath string, v4Addr *net.IPNet, v6Addr *net.IPNet, netLink netlinkwrapper.NetLink,
	ns nswrapper.NS, mtu int, procSys procsyswrapper.ProcSys, log logger.Logger) (netlink.Link, error) {
	if oldHostVeth, err := netLink.LinkByName(hostVethName); err == nil {
		if err = netLink.LinkDel(oldHostVeth); err != nil {
			return nil, errors.Wrapf(err, "setupVeth network: failed to delete old hostVeth %q", hostVethName)
		}
		log.Debugf("Cleaned up old hostVeth: %v\n", hostVethName)
	}
	log.Debugf("v4addr: %v; v6Addr: %v\n", v4Addr, v6Addr)
	createVethContext := newCreateVethPairContext(contVethName, hostVethName, v4Addr, v6Addr, mtu)
	if err := ns.WithNetNSPath(netnsPath, createVethContext.run); err != nil {
		log.Errorf("Failed to setup veth network %v", err)
		return nil, errors.Wrap(err, "setupVeth network: failed to setup veth network")
	}
	hostVeth, err := netLink.LinkByName(hostVethName)
	if err != nil {
		return nil, errors.Wrapf(err, "setupVeth network: failed to find link %q", hostVethName)
	}
	if err := procSys.Set(fmt.Sprintf("net/ipv6/conf/%s/accept_ra", hostVethName), "0"); err != nil {
		if !os.IsNotExist(err) {
			return nil, errors.Wrapf(err, "setupVeth network: failed to disable IPv6 router advertisements")
		}
		log.Debugf("setupVeth network: Ignoring '%s' writing to accept_ra: Assuming kernel lacks IPv6 support", err)
	}
	if err := procSys.Set(fmt.Sprintf("net/ipv6/conf/%s/accept_redirects", hostVethName), "0"); err != nil {
		if !os.IsNotExist(err) {
			return nil, errors.Wrapf(err, "setupVeth network: failed to disable IPv6 ICMP redirects")
		}
		log.Debugf("setupVeth network: Ignoring '%s' writing to accept_redirects: Assuming kernel lacks IPv6 support", err)
	}
	log.Debugf("setupVeth network: disabled IPv6 RA and ICMP redirects on %s", hostVethName)
	if err = netLink.LinkSetUp(hostVeth); err != nil {
		return nil, errors.Wrapf(err, "setupVeth network: failed to set link %q up", hostVethName)
	}
	return hostVeth, nil
}
func (os *linuxNetwork) SetupPodENINetwork(hostVethName string, contVethName string, netnsPath string, v4Addr *net.IPNet,
	v6Addr *net.IPNet, vlanID int, eniMAC string, subnetGW string, parentIfIndex int, mtu int, log logger.Logger) error {
	hostVeth, err := setupVeth(hostVethName, contVethName, netnsPath, v4Addr, v6Addr, os.netLink, os.ns, mtu, os.procSys, log)
	if err != nil {
		return errors.Wrapf(err, "SetupPodENINetwork failed to setup veth pair.")
	}
	vlanTableID := vlanID + 100
	vlanLink := buildVlanLink(vlanID, parentIfIndex, eniMAC)
	if oldVlan, err := os.netLink.LinkByName(vlanLink.Name); err == nil {
		if err = os.netLink.LinkDel(oldVlan); err != nil {
			return errors.Wrapf(err, "SetupPodENINetwork: failed to delete old vlan %s", vlanLink.Name)
		}
		log.Debugf("Cleaned up old vlan: %s", vlanLink.Name)
	}
	oldVlanRule := os.netLink.NewRule()
	oldVlanRule.IifName = hostVethName
	oldVlanRule.Priority = vlanRulePriority
	for {
		if err := os.netLink.RuleDel(oldVlanRule); err != nil {
			if !containsNoSuchRule(err) {
				return errors.Wrapf(err, "SetupPodENINetwork: failed to delete hostveth rule for %s", hostVeth.Attrs().Name)
			}
			break
		}
	}
	err = os.netLink.LinkAdd(vlanLink)
	if err != nil {
		return errors.Wrapf(err, "SetupPodENINetwork: failed to add vlan link.")
	}
	if err = os.netLink.LinkSetUp(vlanLink); err != nil {
		return errors.Wrapf(err, "SetupPodENINetwork: failed to set link %q up", vlanLink.Name)
	}
	routes := buildRoutesForVlan(vlanTableID, vlanLink.Index, net.ParseIP(subnetGW))
	for _, r := range routes {
		if err := os.netLink.RouteReplace(&r); err != nil {
			return errors.Wrapf(err, "SetupPodENINetwork: unable to replace route entry %s via %s", r.Dst.IP.String(), subnetGW)
		}
	}
	var addr *net.IPNet
	if v4Addr != nil {
		addr = v4Addr
	} else if v6Addr != nil {
		addr = v6Addr
	}
	route := netlink.Route{
		LinkIndex: hostVeth.Attrs().Index,
		Scope:     netlink.SCOPE_LINK,
		Dst:       addr,
		Table:     vlanTableID,
	}
	if err := os.netLink.RouteReplace(&route); err != nil {
		return errors.Wrapf(err, "SetupPodENINetwork: unable to add or replace route entry for %s", route.Dst.IP.String())
	}
	log.Debugf("Successfully set host route to be %s/0", route.Dst.IP.String())
	vlanRule := os.netLink.NewRule()
	vlanRule.Table = vlanTableID
	vlanRule.Priority = vlanRulePriority
	vlanRule.IifName = vlanLink.Name
	err = os.netLink.RuleAdd(vlanRule)
	if err != nil && !isRuleExistsError(err) {
		return errors.Wrapf(err, "SetupPodENINetwork: unable to add ip rule for vlan link %s ", vlanLink.Name)
	}
	vlanRule.IifName = hostVeth.Attrs().Name
	err = os.netLink.RuleAdd(vlanRule)
	if err != nil && !isRuleExistsError(err) {
		return errors.Wrapf(err, "SetupPodENINetwork: unable to add ip rule for host veth %s", hostVethName)
	}
	return nil
}
func buildRoutesForVlan(vlanTableID int, vlanIndex int, gw net.IP) []netlink.Route {
	return []netlink.Route{
		{
			LinkIndex: vlanIndex,
			Dst:       &net.IPNet{IP: gw, Mask: net.CIDRMask(32, 32)},
			Scope:     netlink.SCOPE_LINK,
			Table:     vlanTableID,
		},
		{
			LinkIndex: vlanIndex,
			Dst:       &net.IPNet{IP: net.IPv4zero, Mask: net.CIDRMask(0, 32)},
			Scope:     netlink.SCOPE_UNIVERSE,
			Gw:        gw,
			Table:     vlanTableID,
		},
	}
}
func buildVlanLink(vlanID int, parentIfIndex int, eniMAC string) *netlink.Vlan {
	la := netlink.NewLinkAttrs()
	la.Name = fmt.Sprintf("vlan.eth.%d", vlanID)
	la.ParentIndex = parentIfIndex
	la.HardwareAddr, _ = net.ParseMAC(eniMAC)
	return &netlink.Vlan{LinkAttrs: la, VlanId: vlanID}
}
func addContainerRule(netLink netlinkwrapper.NetLink, isToContainer bool, addr *net.IPNet, table int) error {
	if addr == nil {
		return errors.New("can't add container rules without an IP address")
	}
	containerRule := netLink.NewRule()
	if isToContainer {
		containerRule.Dst = addr
		containerRule.Priority = toContainerRulePriority
	} else {
		containerRule.Src = addr
		containerRule.Priority = fromContainerRulePriority
	}
	containerRule.Table = table
	err := netLink.RuleDel(containerRule)
	if err != nil && !containsNoSuchRule(err) {
		return errors.Wrapf(err, "addContainerRule: failed to delete old container rule for %s", addr.String())
	}
	err = netLink.RuleAdd(containerRule)
	if err != nil {
		return errors.Wrapf(err, "addContainerRule: failed to add container rule for %s", addr.String())
	}
	return nil
}
func (os *linuxNetwork) TeardownNS(addr *net.IPNet, deviceNumber int, log logger.Logger) error {
	log.Debugf("TeardownNS: addr %s, deviceNumber %d", addr.String(), deviceNumber)
	return tearDownNS(addr, deviceNumber, os.netLink, log)
}
func tearDownNS(addr *net.IPNet, deviceNumber int, netLink netlinkwrapper.NetLink, log logger.Logger) error {
	if addr == nil {
		return errors.New("can't tear down network namespace with no IP address")
	}
	toContainerRule := netLink.NewRule()
	toContainerRule.Dst = addr
	toContainerRule.Priority = toContainerRulePriority
	err := netLink.RuleDel(toContainerRule)
	if err != nil {
		log.Errorf("Failed to delete toContainer rule for %s err %v", addr.String(), err)
	} else {
		log.Infof("Delete toContainer rule for %s ", addr.String())
	}
	if deviceNumber > 0 {
		err := deleteRuleListBySrc(*addr)
		if err != nil {
			log.Errorf("Failed to delete fromContainer for %s %v", addr.String(), err)
			return errors.Wrapf(err, "delete NS network: failed to delete fromContainer rule for %s", addr.String())
		}
		tableNumber := deviceNumber + 1
		log.Infof("Delete fromContainer rule for %s in table %d", addr.String(), tableNumber)
	}
	addrHostAddr := &net.IPNet{
		IP:   addr.IP,
		Mask: net.CIDRMask(32, 32)}
	if err = netLink.RouteDel(&netlink.Route{
		Scope: netlink.SCOPE_LINK,
		Dst:   addrHostAddr}); err != nil {
		log.Errorf("delete NS network: failed to delete host route for %s, %v", addr.String(), err)
	}
	log.Debug("Tear down of NS complete")
	return nil
}