func (f *eniIPFactory) ListResource() (map[string]types.NetworkResource, error) {
	f.RLock()
	defer f.RUnlock()
	mapping := make(map[string]types.NetworkResource)
	var inUseENIIPs []*types.ENIIP
	for _, e := range f.enis {
		e.lock.Lock()
		for _, eniIP := range e.ips {
			inUseENIIPs = append(inUseENIIPs, eniIP.ENIIP)
		}
		e.lock.Unlock()
	}
	ctx := context.Background()
	macs, err := f.eniFactory.ecs.GetSecondaryENIMACs(ctx)
	if err != nil {
		return nil, err
	}
	for _, mac := range macs {
		ipv4s, ipv6s, err := f.eniFactory.ecs.GetENIIPs(ctx, mac)
		if err != nil {
			if errors.Is(err, apiErr.ErrNotFound) {
				continue
			}
			return nil, err
		}
		ipv4Set := terwayIP.ToIPMap(ipv4s)
		ipv6Set := terwayIP.ToIPMap(ipv6s)
		for _, eniIP := range inUseENIIPs {
			if eniIP.ENI.MAC != mac {
				continue
			}
			var v4, v6 net.IP
			if eniIP.IPSet.IPv4 != nil {
				_, ok := ipv4Set[eniIP.IPSet.IPv4.String()]
				if ok {
					v4 = eniIP.IPSet.IPv4
				}
			}
			if eniIP.IPSet.IPv6 != nil {
				_, ok := ipv6Set[eniIP.IPSet.IPv6.String()]
				if ok {
					v6 = eniIP.IPSet.IPv6
				}
			}
			tmp := types.ENIIP{
				ENI: &types.ENI{
					MAC: mac,
				},
				IPSet: types.IPSet{
					IPv4: v4,
					IPv6: v6,
				},
			}
			mapping[tmp.GetResourceID()] = &tmp
		}
	}
	return mapping, nil
}
ListResource load all eni info from metadata
package daemon
import (
	"context"
	"fmt"
	"net"
	"sort"
	"strings"
	"sync"
	"time"
	"github.com/AliyunContainerService/terway/deviceplugin"
	"github.com/AliyunContainerService/terway/pkg/aliyun"
	apiErr "github.com/AliyunContainerService/terway/pkg/aliyun/errors"
	terwayIP "github.com/AliyunContainerService/terway/pkg/ip"
	"github.com/AliyunContainerService/terway/pkg/ipam"
	"github.com/AliyunContainerService/terway/pkg/logger"
	"github.com/AliyunContainerService/terway/pkg/metric"
	"github.com/AliyunContainerService/terway/pkg/pool"
	"github.com/AliyunContainerService/terway/pkg/tracing"
	"github.com/AliyunContainerService/terway/types"
	"github.com/pkg/errors"
	"github.com/prometheus/client_golang/prometheus"
	corev1 "k8s.io/api/core/v1"
)
var eniIPLog = logger.DefaultLogger
const (
	maxEniOperating = 3
	maxIPBacklog    = 10
)
const (
	eniIPAllocInhibitTimeout = 10 * time.Minute
	typeNameENIIP    = "eniip"
	poolNameENIIP    = "eniip"
	factoryNameENIIP = "eniip"
	tracingKeyENIMaxIP         = "eni_max_ip"
	tracingKeyENICount         = "eni_count"
	tracingKeySecondaryIPCount = "secondary_ip_count"
	commandAudit = "audit"
)
const timeFormat = "2006-01-02 15:04:05"
type eniIPFactory struct {
	name         string
	enableTrunk  bool
	trunkOnEni   string
	eniFactory   *eniFactory
	enis         []*ENI
	maxENI       chan struct{}
	eniMaxIP     int
	eniOperChan  chan struct{}
	ipResultChan chan *ENIIP
	sync.RWMutex
	metricENICount prometheus.Gauge
	ipFamily *types.IPFamily
}
type ENIIP struct {
	*types.ENIIP
	err error
}
type ENI struct {
	lock sync.Mutex
	*types.ENI
	ips       []*ENIIP
	pending   int
	ipBacklog chan struct{}
	ecs       ipam.API
	done      chan struct{}
	ipAllocInhibitExpireAt time.Time
}
func (e *ENI) getIPCountLocked() int {
	return e.pending + len(e.ips)
}
func (e *ENI) allocateWorker(resultChan chan<- *ENIIP) {
	for {
		toAllocate := 0
		select {
		case <-e.done:
			return
		case <-e.ipBacklog:
			toAllocate = 1
		}
		time.Sleep(300 * time.Millisecond)
	popAll:
		for {
			select {
			case <-e.ipBacklog:
				toAllocate++
			default:
				break popAll
			}
		}
		eniIPLog.Debugf("allocate %v ips for eni", toAllocate)
		ips, err := e.ecs.AssignNIPsForENI(context.Background(), e.ENI.ID, e.ENI.MAC, toAllocate)
		eniIPLog.Debugf("allocated ips for eni: eni = %+v, ips = %+v, err = %v", e.ENI, ips, err)
		if err != nil {
			eniIPLog.Errorf("error allocate ips for eni: %v", err)
			metric.ENIIPFactoryIPAllocCount.WithLabelValues(e.MAC, metric.ENIIPAllocActionFail).Add(float64(toAllocate))
			for i := 0; i < toAllocate; i++ {
				resultChan <- &ENIIP{
					ENIIP: &types.ENIIP{
						ENI: e.ENI,
					},
					err: errors.Errorf("error assign ip for ENI: %v", err),
				}
			}
		} else {
			metric.ENIIPFactoryIPAllocCount.WithLabelValues(e.MAC, metric.ENIIPAllocActionSucceed).Add(float64(toAllocate))
			for _, ip := range ips {
				resultChan <- &ENIIP{
					ENIIP: &types.ENIIP{
						ENI:   e.ENI,
						IPSet: ip,
					},
					err: nil,
				}
			}
		}
	}
}
func (f *eniIPFactory) getEnis() ([]*ENI, error) {
	var (
		enis        []*ENI
		pendingEnis []*ENI
	)
	enisLen := len(f.enis)
	if enisLen <= 1 {
		return f.enis, nil
	}
	sort.Slice(f.enis, func(i, j int) bool {
		return f.enis[i].getIPCountLocked() < f.enis[j].getIPCountLocked()
	})
	vSwitches, err := f.eniFactory.GetVSwitches()
	eniIPLog.Infof("adjusted vswitch slice: %+v, original eni slice: %+v", vSwitches, f.enis)
	if err != nil {
		eniIPLog.Errorf("error to get vswitch slice: %v, instead use original eni slice in eniIPFactory: %v", err, f.enis)
		return f.enis, err
	}
	for _, vswitch := range vSwitches {
		for _, eni := range f.enis {
			if eni.ENI == nil {
				pendingEnis = append(pendingEnis, eni)
				continue
			}
			if vswitch == eni.VSwitchID {
				enis = append(enis, eni)
			}
		}
	}
	enis = append(enis, pendingEnis...)
	return enis, nil
}
func (f *eniIPFactory) submit() error {
	f.Lock()
	defer f.Unlock()
	var enis []*ENI
	enis, _ = f.getEnis()
	for _, eni := range enis {
		eniIPLog.Infof("check existing eni: %+v", eni)
		eni.lock.Lock()
		now := time.Now()
		if eni.ENI != nil {
			eniIPLog.Infof("check if the current eni is in the time window for IP allocation inhibition: "+
				"eni = %+v, vsw= %s, now = %s, expireAt = %s", eni, eni.VSwitchID, now.Format(timeFormat), eni.ipAllocInhibitExpireAt.Format(timeFormat))
		}
		if now.Before(eni.ipAllocInhibitExpireAt) && eni.ENI != nil {
			eni.lock.Unlock()
			eniIPLog.Debugf("skip IP allocation: eni = %+v, vsw = %s", eni, eni.VSwitchID)
			continue
		}
		eniIPLog.Debugf("check if the current eni will reach eni IP quota with new pending IP added: "+
			"eni = %+v, eni.pending = %d, len(eni.ips) = %d, eni.MaxIPs = %d", eni, eni.pending, len(eni.ips), f.eniMaxIP)
		if eni.getIPCountLocked() < f.eniMaxIP {
			select {
			case eni.ipBacklog <- struct{}{}:
			default:
				eni.lock.Unlock()
				continue
			}
			eniIPLog.Debugf("submit eniip to the eni backlog: %+v", eni)
			eni.pending++
			eni.lock.Unlock()
			return nil
		}
		eni.lock.Unlock()
	}
	return errors.Errorf("trigger ENIIP throttle, max operating concurrent: %v", maxIPBacklog)
}
func (f *eniIPFactory) popResult() (ip *types.ENIIP, err error) {
	result := <-f.ipResultChan
	eniIPLog.Debugf("pop result from resultChan: %+v", result)
	if result.ENIIP == nil || result.err != nil {
		f.Lock()
		defer f.Unlock()
		if result.ENIIP != nil && result.ENI != nil {
			for _, eni := range f.enis {
				if eni.MAC == result.ENI.MAC {
					eni.pending--
					if strings.Contains(result.err.Error(), apiErr.InvalidVSwitchIDIPNotEnough) {
						eni.ipAllocInhibitExpireAt = time.Now().Add(eniIPAllocInhibitTimeout)
						eniIPLog.Infof("eni's associated vswitch %s has no available IP, set eni ipAllocInhibitExpireAt = %s",
							eni.VSwitchID, eni.ipAllocInhibitExpireAt.Format(timeFormat))
					}
				}
			}
		}
		return nil, errors.Errorf("error allocate ip from eni: %v", result.err)
	}
	f.Lock()
	defer f.Unlock()
	for _, eni := range f.enis {
		if eni.ENI != nil && eni.MAC == result.ENI.MAC {
			eni.pending--
			eni.lock.Lock()
			eni.ips = append(eni.ips, result)
			eni.lock.Unlock()
			metric.ENIIPFactoryIPCount.WithLabelValues(f.name, eni.MAC, fmt.Sprint(eni.MaxIPs)).Inc()
			return result.ENIIP, nil
		}
	}
	return nil, errors.Errorf("unexpected eni ip allocated: %v", result)
}
func (f *eniIPFactory) Create(count int) ([]types.NetworkResource, error) {
	var (
		ipResult []types.NetworkResource
		err      error
		waiting  int
	)
	defer func() {
		if len(ipResult) == 0 {
			eniIPLog.Debugf("create result: %v, error: %v", ipResult, err)
		} else {
			for _, ip := range ipResult {
				eniIPLog.Debugf("create result nil: %+v, error: %v", ip, err)
			}
		}
	}()
	for ; waiting < count; waiting++ {
		err = f.submit()
		if err != nil {
			break
		}
	}
	initENIIPCount := count - waiting
	if initENIIPCount > f.eniMaxIP {
		initENIIPCount = f.eniMaxIP
	}
	if initENIIPCount > maxIPBacklog {
		initENIIPCount = maxIPBacklog
	}
	if initENIIPCount > 0 {
		eniIPLog.Debugf("create eni async, ip count: %+v", initENIIPCount)
		_, err = f.createENIAsync(initENIIPCount)
		if err == nil {
			waiting += initENIIPCount
		} else {
			eniIPLog.Errorf("error create eni async: %+v", err)
		}
	}
	if waiting == 0 {
		return ipResult, errors.Errorf("error submit ip create request: %v", err)
	}
	var ip *types.ENIIP
	for ; waiting > 0; waiting-- {  
		ip, err = f.popResult()
		if err != nil {
			eniIPLog.Errorf("error allocate ip address: %+v", err)
		} else {
			ipResult = append(ipResult, ip)
		}
	}
	if len(ipResult) == 0 {
		return ipResult, errors.Errorf("error allocate ip address: %v", err)
	}
	return ipResult, nil
}
func (f *eniIPFactory) Dispose(res types.NetworkResource) (err error) {
	defer func() {
		eniIPLog.Debugf("dispose result: %v, error: %v", res.GetResourceID(), err != nil)
	}()
	ip := res.(*types.ENIIP)
	var (
		eni   *ENI
		eniip *ENIIP
	)
	f.RLock()
	for _, e := range f.enis {
		if ip.ENI.ID == e.ID {
			eni = e
			e.lock.Lock()
			for _, eip := range e.ips {
				if eip.IPSet.String() == ip.IPSet.String() {
					eniip = eip
				}
			}
			e.lock.Unlock()
		}
	}
	f.RUnlock()
	if eni == nil || eniip == nil {
		return fmt.Errorf("invalid resource to dispose")
	}
	eni.lock.Lock()
	if len(eni.ips) == 1 {
		if f.enableTrunk && eni.Trunk {
			eni.lock.Unlock()
			return fmt.Errorf("trunk ENI %+v will not dispose", eni.ID)
		}
		if eni.pending > 0 {
			eni.lock.Unlock()
			return fmt.Errorf("ENI have pending ips to be allocate")
		}
		eni.pending = eni.MaxIPs
		eni.lock.Unlock()
		f.Lock()
		for i, e := range f.enis {
			if ip.ENI.ID == e.ID {
				close(eni.done)
				f.enis[len(f.enis)-1], f.enis[i] = f.enis[i], f.enis[len(f.enis)-1]
				f.enis = f.enis[:len(f.enis)-1]
				break
			}
		}
		f.metricENICount.Dec()
		f.Unlock()
		f.eniOperChan <- struct{}{}
		err = f.eniFactory.Dispose(ip.ENI)
		<-f.eniOperChan
		if err != nil {
			return fmt.Errorf("error dispose ENI for eniip, %v", err)
		}
		<-f.maxENI
		return nil
	}
	eni.lock.Unlock()
	if ip.ENI.PrimaryIP.IPv4.Equal(ip.IPSet.IPv4) {
		return fmt.Errorf("ip to be release is primary ip of ENI")
	}
	var v4, v6 []net.IP
	if ip.IPSet.IPv4 != nil {
		v4 = append(v4, ip.IPSet.IPv4)
	}
	if ip.IPSet.IPv6 != nil {
		v6 = append(v6, ip.IPSet.IPv6)
	}
	err = f.eniFactory.ecs.UnAssignIPsForENI(context.Background(), ip.ENI.ID, ip.ENI.MAC, v4, v6)
	if err != nil {
		return fmt.Errorf("error unassign eniip, %v", err)
	}
	eni.lock.Lock()
	for i, e := range eni.ips {
		if e.IPSet.IPv4.Equal(eniip.IPSet.IPv4) {
			eni.ips[len(eni.ips)-1], eni.ips[i] = eni.ips[i], eni.ips[len(eni.ips)-1]
			eni.ips = eni.ips[:len(eni.ips)-1]
			break
		}
	}
	eni.lock.Unlock()
	metric.ENIIPFactoryIPCount.WithLabelValues(f.name, eni.MAC, fmt.Sprint(eni.MaxIPs)).Dec()
	return nil
}
func (f *eniIPFactory) Check(res types.NetworkResource) error {
	eniIP, ok := res.(*types.ENIIP)
	if !ok {
		return fmt.Errorf("unsupported type %T", res)
	}
	ipv4, ipv6, err := f.eniFactory.ecs.GetENIIPs(context.Background(), eniIP.ENI.MAC)
	if err != nil {
		return err
	}
	if eniIP.IPSet.IPv4 != nil {
		if !terwayIP.IPsIntersect([]net.IP{eniIP.IPSet.IPv4}, ipv4) {
			return apiErr.ErrNotFound
		}
	}
	if eniIP.IPSet.IPv6 != nil {
		if !terwayIP.IPsIntersect([]net.IP{eniIP.IPSet.IPv6}, ipv6) {
			return apiErr.ErrNotFound
		}
	}
	return nil
}