Change:
Is this a BUG REPORT or FEATURE REQUEST?: /kind bug What happened: After running some tests, we're seeing that the combined usage of kubelet + docker can be around 2Gi. We need to be able to account for this in scheduling to avoid crashing these services. The first step towards this is using kube-reserved and system-reserved to avoid oversubscribing the virtual memory space available on the Windows nodes with too many containers. What you expected to happen: It should reduce node allocatable memory by kube-reserved + system-reserved. How to reproduce it (as minimally and precisely as possible): This logs an error in the kubelet log such as: code: Anything else we need to know?: /milestone v1.13 /assign feiskyer /sig windows This blocks a PR where I want to set it by default to 2Gi Environment: Kubernetes version (use ): v1.11.3 (also v1.12) Cloud provider or hardware configuration: Azure OS (e.g. from /etc/os-release): Windows Server version 1803 Install tools: acs-engine
You must be a member of the github team to set the milestone. <detailsIn response to : Is this a BUG REPORT or FEATURE REQUEST?: What happened: What you expected to happen: This logs an error in the kubelet log such as: code: /milestone v1.13 /assign feiskyer Kubernetes version (use ): v1.11.3 (also v1.12) Cloud provider or hardware configuration: Azure OS (e.g. from /etc/os-release): Windows Server version 1803 Install tools: acs-engine Instructions for interacting with me using PR comments are available . If you have questions or suggestions related to my behavior, please file an issue against the repository. </details
/milestone v1.13
/priority important-soon

Discussion:
"container_manager_unsupported.go", "container_manager_windows.go", "fake_internal_container_lifecycle.go", <ins> "helpers.go", </ins> "helpers_linux.go", "helpers_unsupported.go", "internal_container_lifecycle.go", <del> "node_container_manager.go", </del> <ins> "node_container_manager_linux.go", </ins> "pod_container_manager_linux.go", "pod_container_manager_stub.go", "pod_container_manager_unsupported.go",

"cgroup_manager_test.go", "container_manager_linux_test.go", "helpers_linux_test.go", <del> "node_container_manager_test.go", </del> <ins> "node_container_manager_linux_test.go", </ins> "pod_container_manager_linux_test.go", ], embed = [":go_default_library"],

limitations under the License. */ <ins> // containerManagerImpl implements container manager on Windows. // Only GetNodeAllocatableReservation() and GetCapacity() are implemented now. </ins> package cm import ( <del> "github.com/golang/glog" </del> <ins> "fmt" "github.com/golang/glog" </ins> "k8s.io/api/core/v1" <ins> "k8s.io/apimachinery/pkg/api/resource" utilfeature "k8s.io/apiserver/pkg/util/feature" </ins> "k8s.io/client-go/tools/record" <ins> kubefeatures "k8s.io/kubernetes/pkg/features" </ins> internalapi "k8s.io/kubernetes/pkg/kubelet/apis/cri" "k8s.io/kubernetes/pkg/kubelet/cadvisor" <ins> "k8s.io/kubernetes/pkg/kubelet/cm/cpumanager" </ins> "k8s.io/kubernetes/pkg/kubelet/config" <ins> kubecontainer "k8s.io/kubernetes/pkg/kubelet/container" "k8s.io/kubernetes/pkg/kubelet/lifecycle" </ins> "k8s.io/kubernetes/pkg/kubelet/status" <ins> "k8s.io/kubernetes/pkg/kubelet/util/pluginwatcher" schedulercache "k8s.io/kubernetes/pkg/scheduler/cache" </ins> "k8s.io/kubernetes/pkg/util/mount" ) type containerManagerImpl struct { <del> containerManagerStub </del> <ins> // Capacity of this node. capacity v1.ResourceList // Interface for cadvisor. cadvisorInterface cadvisor.Interface // Config of this node. nodeConfig NodeConfig </ins> } <del> var _ ContainerManager = &containerManagerImpl{} </del> <ins> func (cm *containerManagerImpl) Start(node *v1.Node, activePods ActivePodsFunc, sourcesReady config.SourcesReady, podStatusProvider status.PodStatusProvider, runtimeService internalapi.RuntimeService) error { glog.V(2).Infof("Starting Windows container manager") if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.LocalStorageCapacityIsolation) { rootfs, err := cm.cadvisorInterface.RootFsInfo() if err != nil { return fmt.Errorf("failed to get rootfs info: %v", err) } for rName, rCap := range cadvisor.EphemeralStorageCapacityFromFsInfo(rootfs) { cm.capacity[rName] = rCap } } </ins> <del> func (cm *containerManagerImpl) Start(_ *v1.Node, _ ActivePodsFunc, _ config.SourcesReady, _ status.PodStatusProvider, _ internalapi.RuntimeService) error { glog.V(2).Infof("Starting Windows stub container manager") </del> return nil } <ins> // NewContainerManager creates windows container manager. </ins> func NewContainerManager(mountUtil mount.Interface, cadvisorInterface cadvisor.Interface, nodeConfig NodeConfig, failSwapOn bool, devicePluginEnabled bool, recorder record.EventRecorder) (ContainerManager, error) { <del> return &containerManagerImpl{}, nil </del> <ins> var capacity = v1.ResourceList{} // It is safe to invoke `MachineInfo` on cAdvisor before logically initializing cAdvisor here because // machine info is computed and cached once as part of cAdvisor object creation. // But `RootFsInfo` and `ImagesFsInfo` are not available at this moment so they will be called later during manager starts machineInfo, err := cadvisorInterface.MachineInfo() if err != nil { return nil, err } capacity = cadvisor.CapacityFromMachineInfo(machineInfo) return &containerManagerImpl{ capacity:          capacity, nodeConfig:        nodeConfig, cadvisorInterface: cadvisorInterface, }, nil } func (cm *containerManagerImpl) SystemCgroupsLimit() v1.ResourceList { return v1.ResourceList{} } func (cm *containerManagerImpl) GetNodeConfig() NodeConfig { return NodeConfig{} } func (cm *containerManagerImpl) GetMountedSubsystems() *CgroupSubsystems { return &CgroupSubsystems{} } func (cm *containerManagerImpl) GetQOSContainersInfo() QOSContainersInfo { return QOSContainersInfo{} } func (cm *containerManagerImpl) UpdateQOSCgroups() error { return nil } func (cm *containerManagerImpl) Status() Status { return Status{} } func (cm *containerManagerImpl) GetNodeAllocatableReservation() v1.ResourceList { evictionReservation := hardEvictionReservation(cm.nodeConfig.HardEvictionThresholds, cm.capacity) result := make(v1.ResourceList) for k := range cm.capacity { value := resource.NewQuantity(0, resource.DecimalSI) if cm.nodeConfig.SystemReserved != nil { value.Add(cm.nodeConfig.SystemReserved[k]) } if cm.nodeConfig.KubeReserved != nil { value.Add(cm.nodeConfig.KubeReserved[k]) } if evictionReservation != nil { value.Add(evictionReservation[k]) } if !value.IsZero() { result[k] = *value } } return result } func (cm *containerManagerImpl) GetCapacity() v1.ResourceList { return cm.capacity } func (cm *containerManagerImpl) GetPluginRegistrationHandler() pluginwatcher.PluginHandler { return nil } func (cm *containerManagerImpl) GetDevicePluginResourceCapacity() (v1.ResourceList, v1.ResourceList, []string) { return nil, nil, []string{} } func (cm *containerManagerImpl) NewPodContainerManager() PodContainerManager { return &podContainerManagerStub{} } func (cm *containerManagerImpl) GetResources(pod *v1.Pod, container *v1.Container) (*kubecontainer.RunContainerOptions, error) { return &kubecontainer.RunContainerOptions{}, nil } func (cm *containerManagerImpl) UpdatePluginResources(*schedulercache.NodeInfo, *lifecycle.PodAdmitAttributes) error { return nil } func (cm *containerManagerImpl) InternalContainerLifecycle() InternalContainerLifecycle { return &internalContainerLifecycleImpl{cpumanager.NewFakeManager()} } func (cm *containerManagerImpl) GetPodCgroupRoot() string { return "" </ins> }

<ins> /* Copyright 2018 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package cm import ( "k8s.io/api/core/v1" evictionapi "k8s.io/kubernetes/pkg/kubelet/eviction/api" ) // hardEvictionReservation returns a resourcelist that includes reservation of resources based on hard eviction thresholds. func hardEvictionReservation(thresholds []evictionapi.Threshold, capacity v1.ResourceList) v1.ResourceList { if len(thresholds) == 0 { return nil } ret := v1.ResourceList{} for _, threshold := range thresholds { if threshold.Operator != evictionapi.OpLessThan { continue } switch threshold.Signal { case evictionapi.SignalMemoryAvailable: memoryCapacity := capacity[v1.ResourceMemory] value := evictionapi.GetThresholdQuantity(threshold.Value, &memoryCapacity) ret[v1.ResourceMemory] = *value case evictionapi.SignalNodeFsAvailable: storageCapacity := capacity[v1.ResourceEphemeralStorage] value := evictionapi.GetThresholdQuantity(threshold.Value, &storageCapacity) ret[v1.ResourceEphemeralStorage] = *value } } return ret } </ins>

<del> // +build linux /* Copyright 2017 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package cm import ( "fmt" "strings" "time" "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/types" utilfeature "k8s.io/apiserver/pkg/util/feature" kubefeatures "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/kubelet/events" evictionapi "k8s.io/kubernetes/pkg/kubelet/eviction/api" kubetypes "k8s.io/kubernetes/pkg/kubelet/types" ) const ( defaultNodeAllocatableCgroupName = "kubepods" ) //createNodeAllocatableCgroups creates Node Allocatable Cgroup when CgroupsPerQOS flag is specified as true func (cm *containerManagerImpl) createNodeAllocatableCgroups() error { cgroupConfig := &CgroupConfig{ Name: cm.cgroupRoot, // The default limits for cpu shares can be very low which can lead to CPU starvation for pods. ResourceParameters: getCgroupConfig(cm.capacity), } if cm.cgroupManager.Exists(cgroupConfig.Name) { return nil } if err := cm.cgroupManager.Create(cgroupConfig); err != nil { glog.Errorf("Failed to create %q cgroup", cm.cgroupRoot) return err } return nil } // enforceNodeAllocatableCgroups enforce Node Allocatable Cgroup settings. func (cm *containerManagerImpl) enforceNodeAllocatableCgroups() error { nc := cm.NodeConfig.NodeAllocatableConfig // We need to update limits on node allocatable cgroup no matter what because // default cpu shares on cgroups are low and can cause cpu starvation. nodeAllocatable := cm.capacity // Use Node Allocatable limits instead of capacity if the user requested enforcing node allocatable. if cm.CgroupsPerQOS && nc.EnforceNodeAllocatable.Has(kubetypes.NodeAllocatableEnforcementKey) { nodeAllocatable = cm.getNodeAllocatableAbsolute() } glog.V(4).Infof("Attempting to enforce Node Allocatable with config: %+v", nc) cgroupConfig := &CgroupConfig{ Name:               cm.cgroupRoot, ResourceParameters: getCgroupConfig(nodeAllocatable), } // Using ObjectReference for events as the node maybe not cached; refer to #42701 for detail. nodeRef := &v1.ObjectReference{ Kind:      "Node", Name:      cm.nodeInfo.Name, UID:       types.UID(cm.nodeInfo.Name), Namespace: "", } // If Node Allocatable is enforced on a node that has not been drained or is updated on an existing node to a lower value, // existing memory usage across pods might be higher than current Node Allocatable Memory Limits. // Pod Evictions are expected to bring down memory usage to below Node Allocatable limits. // Until evictions happen retry cgroup updates. // Update limits on non root cgroup-root to be safe since the default limits for CPU can be too low. // Check if cgroupRoot is set to a non-empty value (empty would be the root container) if len(cm.cgroupRoot) > 0 { go func() { for { err := cm.cgroupManager.Update(cgroupConfig) if err == nil { cm.recorder.Event(nodeRef, v1.EventTypeNormal, events.SuccessfulNodeAllocatableEnforcement, "Updated Node Allocatable limit across pods") return } message := fmt.Sprintf("Failed to update Node Allocatable Limits %q: %v", cm.cgroupRoot, err) cm.recorder.Event(nodeRef, v1.EventTypeWarning, events.FailedNodeAllocatableEnforcement, message) time.Sleep(time.Minute) } }() } // Now apply kube reserved and system reserved limits if required. if nc.EnforceNodeAllocatable.Has(kubetypes.SystemReservedEnforcementKey) { glog.V(2).Infof("Enforcing System reserved on cgroup %q with limits: %+v", nc.SystemReservedCgroupName, nc.SystemReserved) if err := enforceExistingCgroup(cm.cgroupManager, ParseCgroupfsToCgroupName(nc.SystemReservedCgroupName), nc.SystemReserved); err != nil { message := fmt.Sprintf("Failed to enforce System Reserved Cgroup Limits on %q: %v", nc.SystemReservedCgroupName, err) cm.recorder.Event(nodeRef, v1.EventTypeWarning, events.FailedNodeAllocatableEnforcement, message) return fmt.Errorf(message) } cm.recorder.Eventf(nodeRef, v1.EventTypeNormal, events.SuccessfulNodeAllocatableEnforcement, "Updated limits on system reserved cgroup %v", nc.SystemReservedCgroupName) } if nc.EnforceNodeAllocatable.Has(kubetypes.KubeReservedEnforcementKey) { glog.V(2).Infof("Enforcing kube reserved on cgroup %q with limits: %+v", nc.KubeReservedCgroupName, nc.KubeReserved) if err := enforceExistingCgroup(cm.cgroupManager, ParseCgroupfsToCgroupName(nc.KubeReservedCgroupName), nc.KubeReserved); err != nil { message := fmt.Sprintf("Failed to enforce Kube Reserved Cgroup Limits on %q: %v", nc.KubeReservedCgroupName, err) cm.recorder.Event(nodeRef, v1.EventTypeWarning, events.FailedNodeAllocatableEnforcement, message) return fmt.Errorf(message) } cm.recorder.Eventf(nodeRef, v1.EventTypeNormal, events.SuccessfulNodeAllocatableEnforcement, "Updated limits on kube reserved cgroup %v", nc.KubeReservedCgroupName) } return nil } // enforceExistingCgroup updates the limits `rl` on existing cgroup `cName` using `cgroupManager` interface. func enforceExistingCgroup(cgroupManager CgroupManager, cName CgroupName, rl v1.ResourceList) error { cgroupConfig := &CgroupConfig{ Name:               cName, ResourceParameters: getCgroupConfig(rl), } if cgroupConfig.ResourceParameters == nil { return fmt.Errorf("%q cgroup is not config properly", cgroupConfig.Name) } glog.V(4).Infof("Enforcing limits on cgroup %q with %d cpu shares and %d bytes of memory", cName, cgroupConfig.ResourceParameters.CpuShares, cgroupConfig.ResourceParameters.Memory) if !cgroupManager.Exists(cgroupConfig.Name) { return fmt.Errorf("%q cgroup does not exist", cgroupConfig.Name) } if err := cgroupManager.Update(cgroupConfig); err != nil { return err } return nil } // getCgroupConfig returns a ResourceConfig object that can be used to create or update cgroups via CgroupManager interface. func getCgroupConfig(rl v1.ResourceList) *ResourceConfig { // TODO(vishh): Set CPU Quota if necessary. if rl == nil { return nil } var rc ResourceConfig if q, exists := rl[v1.ResourceMemory]; exists { // Memory is defined in bytes. val := q.Value() rc.Memory = &val } if q, exists := rl[v1.ResourceCPU]; exists { // CPU is defined in milli-cores. val := MilliCPUToShares(q.MilliValue()) rc.CpuShares = &val } if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.HugePages) { rc.HugePageLimit = HugePageLimits(rl) } return &rc } // getNodeAllocatableAbsolute returns the absolute value of Node Allocatable which is primarily useful for enforcement. // Note that not all resources that are available on the node are included in the returned list of resources. // Returns a ResourceList. func (cm *containerManagerImpl) getNodeAllocatableAbsolute() v1.ResourceList { result := make(v1.ResourceList) for k, v := range cm.capacity { value := *(v.Copy()) if cm.NodeConfig.SystemReserved != nil { value.Sub(cm.NodeConfig.SystemReserved[k]) } if cm.NodeConfig.KubeReserved != nil { value.Sub(cm.NodeConfig.KubeReserved[k]) } if value.Sign() < 0 { // Negative Allocatable resources don't make sense. value.Set(0) } result[k] = value } return result } // GetNodeAllocatableReservation returns amount of compute or storage resource that have to be reserved on this node from scheduling. func (cm *containerManagerImpl) GetNodeAllocatableReservation() v1.ResourceList { evictionReservation := hardEvictionReservation(cm.HardEvictionThresholds, cm.capacity) result := make(v1.ResourceList) for k := range cm.capacity { value := resource.NewQuantity(0, resource.DecimalSI) if cm.NodeConfig.SystemReserved != nil { value.Add(cm.NodeConfig.SystemReserved[k]) } if cm.NodeConfig.KubeReserved != nil { value.Add(cm.NodeConfig.KubeReserved[k]) } if evictionReservation != nil { value.Add(evictionReservation[k]) } if !value.IsZero() { result[k] = *value } } return result } // hardEvictionReservation returns a resourcelist that includes reservation of resources based on hard eviction thresholds. func hardEvictionReservation(thresholds []evictionapi.Threshold, capacity v1.ResourceList) v1.ResourceList { if len(thresholds) == 0 { return nil } ret := v1.ResourceList{} for _, threshold := range thresholds { if threshold.Operator != evictionapi.OpLessThan { continue } switch threshold.Signal { case evictionapi.SignalMemoryAvailable: memoryCapacity := capacity[v1.ResourceMemory] value := evictionapi.GetThresholdQuantity(threshold.Value, &memoryCapacity) ret[v1.ResourceMemory] = *value case evictionapi.SignalNodeFsAvailable: storageCapacity := capacity[v1.ResourceEphemeralStorage] value := evictionapi.GetThresholdQuantity(threshold.Value, &storageCapacity) ret[v1.ResourceEphemeralStorage] = *value } } return ret } // validateNodeAllocatable ensures that the user specified Node Allocatable Configuration doesn't reserve more than the node capacity. // Returns error if the configuration is invalid, nil otherwise. func (cm *containerManagerImpl) validateNodeAllocatable() error { var errors []string nar := cm.GetNodeAllocatableReservation() for k, v := range nar { value := cm.capacity[k].DeepCopy() value.Sub(v) if value.Sign() < 0 { errors = append(errors, fmt.Sprintf("Resource %q has an allocatable of %v, capacity of %v", k, v, value)) } } if len(errors) > 0 { return fmt.Errorf("Invalid Node Allocatable configuration. %s", strings.Join(errors, " ")) } return nil } </del>

<ins> // +build linux /* Copyright 2017 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package cm import ( "fmt" "strings" "time" "github.com/golang/glog" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/types" utilfeature "k8s.io/apiserver/pkg/util/feature" kubefeatures "k8s.io/kubernetes/pkg/features" "k8s.io/kubernetes/pkg/kubelet/events" kubetypes "k8s.io/kubernetes/pkg/kubelet/types" ) const ( defaultNodeAllocatableCgroupName = "kubepods" ) //createNodeAllocatableCgroups creates Node Allocatable Cgroup when CgroupsPerQOS flag is specified as true func (cm *containerManagerImpl) createNodeAllocatableCgroups() error { cgroupConfig := &CgroupConfig{ Name: cm.cgroupRoot, // The default limits for cpu shares can be very low which can lead to CPU starvation for pods. ResourceParameters: getCgroupConfig(cm.capacity), } if cm.cgroupManager.Exists(cgroupConfig.Name) { return nil } if err := cm.cgroupManager.Create(cgroupConfig); err != nil { glog.Errorf("Failed to create %q cgroup", cm.cgroupRoot) return err } return nil } // enforceNodeAllocatableCgroups enforce Node Allocatable Cgroup settings. func (cm *containerManagerImpl) enforceNodeAllocatableCgroups() error { nc := cm.NodeConfig.NodeAllocatableConfig // We need to update limits on node allocatable cgroup no matter what because // default cpu shares on cgroups are low and can cause cpu starvation. nodeAllocatable := cm.capacity // Use Node Allocatable limits instead of capacity if the user requested enforcing node allocatable. if cm.CgroupsPerQOS && nc.EnforceNodeAllocatable.Has(kubetypes.NodeAllocatableEnforcementKey) { nodeAllocatable = cm.getNodeAllocatableAbsolute() } glog.V(4).Infof("Attempting to enforce Node Allocatable with config: %+v", nc) cgroupConfig := &CgroupConfig{ Name:               cm.cgroupRoot, ResourceParameters: getCgroupConfig(nodeAllocatable), } // Using ObjectReference for events as the node maybe not cached; refer to #42701 for detail. nodeRef := &v1.ObjectReference{ Kind:      "Node", Name:      cm.nodeInfo.Name, UID:       types.UID(cm.nodeInfo.Name), Namespace: "", } // If Node Allocatable is enforced on a node that has not been drained or is updated on an existing node to a lower value, // existing memory usage across pods might be higher than current Node Allocatable Memory Limits. // Pod Evictions are expected to bring down memory usage to below Node Allocatable limits. // Until evictions happen retry cgroup updates. // Update limits on non root cgroup-root to be safe since the default limits for CPU can be too low. // Check if cgroupRoot is set to a non-empty value (empty would be the root container) if len(cm.cgroupRoot) > 0 { go func() { for { err := cm.cgroupManager.Update(cgroupConfig) if err == nil { cm.recorder.Event(nodeRef, v1.EventTypeNormal, events.SuccessfulNodeAllocatableEnforcement, "Updated Node Allocatable limit across pods") return } message := fmt.Sprintf("Failed to update Node Allocatable Limits %q: %v", cm.cgroupRoot, err) cm.recorder.Event(nodeRef, v1.EventTypeWarning, events.FailedNodeAllocatableEnforcement, message) time.Sleep(time.Minute) } }() } // Now apply kube reserved and system reserved limits if required. if nc.EnforceNodeAllocatable.Has(kubetypes.SystemReservedEnforcementKey) { glog.V(2).Infof("Enforcing System reserved on cgroup %q with limits: %+v", nc.SystemReservedCgroupName, nc.SystemReserved) if err := enforceExistingCgroup(cm.cgroupManager, ParseCgroupfsToCgroupName(nc.SystemReservedCgroupName), nc.SystemReserved); err != nil { message := fmt.Sprintf("Failed to enforce System Reserved Cgroup Limits on %q: %v", nc.SystemReservedCgroupName, err) cm.recorder.Event(nodeRef, v1.EventTypeWarning, events.FailedNodeAllocatableEnforcement, message) return fmt.Errorf(message) } cm.recorder.Eventf(nodeRef, v1.EventTypeNormal, events.SuccessfulNodeAllocatableEnforcement, "Updated limits on system reserved cgroup %v", nc.SystemReservedCgroupName) } if nc.EnforceNodeAllocatable.Has(kubetypes.KubeReservedEnforcementKey) { glog.V(2).Infof("Enforcing kube reserved on cgroup %q with limits: %+v", nc.KubeReservedCgroupName, nc.KubeReserved) if err := enforceExistingCgroup(cm.cgroupManager, ParseCgroupfsToCgroupName(nc.KubeReservedCgroupName), nc.KubeReserved); err != nil { message := fmt.Sprintf("Failed to enforce Kube Reserved Cgroup Limits on %q: %v", nc.KubeReservedCgroupName, err) cm.recorder.Event(nodeRef, v1.EventTypeWarning, events.FailedNodeAllocatableEnforcement, message) return fmt.Errorf(message) } cm.recorder.Eventf(nodeRef, v1.EventTypeNormal, events.SuccessfulNodeAllocatableEnforcement, "Updated limits on kube reserved cgroup %v", nc.KubeReservedCgroupName) } return nil } // enforceExistingCgroup updates the limits `rl` on existing cgroup `cName` using `cgroupManager` interface. func enforceExistingCgroup(cgroupManager CgroupManager, cName CgroupName, rl v1.ResourceList) error { cgroupConfig := &CgroupConfig{ Name:               cName, ResourceParameters: getCgroupConfig(rl), } if cgroupConfig.ResourceParameters == nil { return fmt.Errorf("%q cgroup is not config properly", cgroupConfig.Name) } glog.V(4).Infof("Enforcing limits on cgroup %q with %d cpu shares and %d bytes of memory", cName, cgroupConfig.ResourceParameters.CpuShares, cgroupConfig.ResourceParameters.Memory) if !cgroupManager.Exists(cgroupConfig.Name) { return fmt.Errorf("%q cgroup does not exist", cgroupConfig.Name) } if err := cgroupManager.Update(cgroupConfig); err != nil { return err } return nil } // getCgroupConfig returns a ResourceConfig object that can be used to create or update cgroups via CgroupManager interface. func getCgroupConfig(rl v1.ResourceList) *ResourceConfig { // TODO(vishh): Set CPU Quota if necessary. if rl == nil { return nil } var rc ResourceConfig if q, exists := rl[v1.ResourceMemory]; exists { // Memory is defined in bytes. val := q.Value() rc.Memory = &val } if q, exists := rl[v1.ResourceCPU]; exists { // CPU is defined in milli-cores. val := MilliCPUToShares(q.MilliValue()) rc.CpuShares = &val } if utilfeature.DefaultFeatureGate.Enabled(kubefeatures.HugePages) { rc.HugePageLimit = HugePageLimits(rl) } return &rc } // getNodeAllocatableAbsolute returns the absolute value of Node Allocatable which is primarily useful for enforcement. // Note that not all resources that are available on the node are included in the returned list of resources. // Returns a ResourceList. func (cm *containerManagerImpl) getNodeAllocatableAbsolute() v1.ResourceList { result := make(v1.ResourceList) for k, v := range cm.capacity { value := *(v.Copy()) if cm.NodeConfig.SystemReserved != nil { value.Sub(cm.NodeConfig.SystemReserved[k]) } if cm.NodeConfig.KubeReserved != nil { value.Sub(cm.NodeConfig.KubeReserved[k]) } if value.Sign() < 0 { // Negative Allocatable resources don't make sense. value.Set(0) } result[k] = value } return result } // GetNodeAllocatableReservation returns amount of compute or storage resource that have to be reserved on this node from scheduling. func (cm *containerManagerImpl) GetNodeAllocatableReservation() v1.ResourceList { evictionReservation := hardEvictionReservation(cm.HardEvictionThresholds, cm.capacity) result := make(v1.ResourceList) for k := range cm.capacity { value := resource.NewQuantity(0, resource.DecimalSI) if cm.NodeConfig.SystemReserved != nil { value.Add(cm.NodeConfig.SystemReserved[k]) } if cm.NodeConfig.KubeReserved != nil { value.Add(cm.NodeConfig.KubeReserved[k]) } if evictionReservation != nil { value.Add(evictionReservation[k]) } if !value.IsZero() { result[k] = *value } } return result } // validateNodeAllocatable ensures that the user specified Node Allocatable Configuration doesn't reserve more than the node capacity. // Returns error if the configuration is invalid, nil otherwise. func (cm *containerManagerImpl) validateNodeAllocatable() error { var errors []string nar := cm.GetNodeAllocatableReservation() for k, v := range nar { value := cm.capacity[k].DeepCopy() value.Sub(v) if value.Sign() < 0 { errors = append(errors, fmt.Sprintf("Resource %q has an allocatable of %v, capacity of %v", k, v, value)) } } if len(errors) > 0 { return fmt.Errorf("Invalid Node Allocatable configuration. %s", strings.Join(errors, " ")) } return nil } </ins>

<ins> // +build linux /* Copyright 2017 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package cm import ( "testing" "github.com/stretchr/testify/assert" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" evictionapi "k8s.io/kubernetes/pkg/kubelet/eviction/api" ) func TestNodeAllocatableReservationForScheduling(t *testing.T) { memoryEvictionThreshold := resource.MustParse("100Mi") cpuMemCases := []struct { kubeReserved   v1.ResourceList systemReserved v1.ResourceList expected       v1.ResourceList capacity       v1.ResourceList hardThreshold  evictionapi.ThresholdValue }{ { kubeReserved:   getResourceList("100m", "100Mi"), systemReserved: getResourceList("50m", "50Mi"), capacity:       getResourceList("10", "10Gi"), expected:       getResourceList("150m", "150Mi"), }, { kubeReserved:   getResourceList("100m", "100Mi"), systemReserved: getResourceList("50m", "50Mi"), hardThreshold: evictionapi.ThresholdValue{ Quantity: &memoryEvictionThreshold, }, capacity: getResourceList("10", "10Gi"), expected: getResourceList("150m", "250Mi"), }, { kubeReserved:   getResourceList("100m", "100Mi"), systemReserved: getResourceList("50m", "50Mi"), capacity:       getResourceList("10", "10Gi"), hardThreshold: evictionapi.ThresholdValue{ Percentage: 0.05, }, expected: getResourceList("150m", "694157320"), }, { kubeReserved:   v1.ResourceList{}, systemReserved: v1.ResourceList{}, capacity:       getResourceList("10", "10Gi"), expected:       getResourceList("", ""), }, { kubeReserved:   getResourceList("", "100Mi"), systemReserved: getResourceList("50m", "50Mi"), capacity:       getResourceList("10", "10Gi"), expected:       getResourceList("50m", "150Mi"), }, { kubeReserved:   getResourceList("50m", "100Mi"), systemReserved: getResourceList("", "50Mi"), capacity:       getResourceList("10", "10Gi"), expected:       getResourceList("50m", "150Mi"), }, { kubeReserved:   getResourceList("", "100Mi"), systemReserved: getResourceList("", "50Mi"), capacity:       getResourceList("10", ""), expected:       getResourceList("", "150Mi"), }, } for idx, tc := range cpuMemCases { nc := NodeConfig{ NodeAllocatableConfig: NodeAllocatableConfig{ KubeReserved:   tc.kubeReserved, SystemReserved: tc.systemReserved, HardEvictionThresholds: []evictionapi.Threshold{ { Signal:   evictionapi.SignalMemoryAvailable, Operator: evictionapi.OpLessThan, Value:    tc.hardThreshold, }, }, }, } cm := &containerManagerImpl{ NodeConfig: nc, capacity:   tc.capacity, } for k, v := range cm.GetNodeAllocatableReservation() { expected, exists := tc.expected[k] assert.True(t, exists, "test case %d expected resource %q", idx+1, k) assert.Equal(t, expected.MilliValue(), v.MilliValue(), "test case %d failed for resource %q", idx+1, k) } } ephemeralStorageEvictionThreshold := resource.MustParse("100Mi") ephemeralStorageTestCases := []struct { kubeReserved  v1.ResourceList expected      v1.ResourceList capacity      v1.ResourceList hardThreshold evictionapi.ThresholdValue }{ { kubeReserved: getEphemeralStorageResourceList("100Mi"), capacity:     getEphemeralStorageResourceList("10Gi"), expected:     getEphemeralStorageResourceList("100Mi"), }, { kubeReserved: getEphemeralStorageResourceList("100Mi"), hardThreshold: evictionapi.ThresholdValue{ Quantity: &ephemeralStorageEvictionThreshold, }, capacity: getEphemeralStorageResourceList("10Gi"), expected: getEphemeralStorageResourceList("200Mi"), }, { kubeReserved: getEphemeralStorageResourceList("150Mi"), capacity:     getEphemeralStorageResourceList("10Gi"), hardThreshold: evictionapi.ThresholdValue{ Percentage: 0.05, }, expected: getEphemeralStorageResourceList("694157320"), }, { kubeReserved: v1.ResourceList{}, capacity:     getEphemeralStorageResourceList("10Gi"), expected:     getEphemeralStorageResourceList(""), }, } for idx, tc := range ephemeralStorageTestCases { nc := NodeConfig{ NodeAllocatableConfig: NodeAllocatableConfig{ KubeReserved: tc.kubeReserved, HardEvictionThresholds: []evictionapi.Threshold{ { Signal:   evictionapi.SignalNodeFsAvailable, Operator: evictionapi.OpLessThan, Value:    tc.hardThreshold, }, }, }, } cm := &containerManagerImpl{ NodeConfig: nc, capacity:   tc.capacity, } for k, v := range cm.GetNodeAllocatableReservation() { expected, exists := tc.expected[k] assert.True(t, exists, "test case %d expected resource %q", idx+1, k) assert.Equal(t, expected.MilliValue(), v.MilliValue(), "test case %d failed for resource %q", idx+1, k) } } } func TestNodeAllocatableForEnforcement(t *testing.T) { memoryEvictionThreshold := resource.MustParse("100Mi") testCases := []struct { kubeReserved   v1.ResourceList systemReserved v1.ResourceList capacity       v1.ResourceList expected       v1.ResourceList hardThreshold  evictionapi.ThresholdValue }{ { kubeReserved:   getResourceList("100m", "100Mi"), systemReserved: getResourceList("50m", "50Mi"), capacity:       getResourceList("10", "10Gi"), expected:       getResourceList("9850m", "10090Mi"), }, { kubeReserved:   getResourceList("100m", "100Mi"), systemReserved: getResourceList("50m", "50Mi"), hardThreshold: evictionapi.ThresholdValue{ Quantity: &memoryEvictionThreshold, }, capacity: getResourceList("10", "10Gi"), expected: getResourceList("9850m", "10090Mi"), }, { kubeReserved:   getResourceList("100m", "100Mi"), systemReserved: getResourceList("50m", "50Mi"), hardThreshold: evictionapi.ThresholdValue{ Percentage: 0.05, }, capacity: getResourceList("10", "10Gi"), expected: getResourceList("9850m", "10090Mi"), }, { kubeReserved:   v1.ResourceList{}, systemReserved: v1.ResourceList{}, capacity:       getResourceList("10", "10Gi"), expected:       getResourceList("10", "10Gi"), }, { kubeReserved:   getResourceList("", "100Mi"), systemReserved: getResourceList("50m", "50Mi"), capacity:       getResourceList("10", "10Gi"), expected:       getResourceList("9950m", "10090Mi"), }, { kubeReserved:   getResourceList("50m", "100Mi"), systemReserved: getResourceList("", "50Mi"), capacity:       getResourceList("10", "10Gi"), expected:       getResourceList("9950m", "10090Mi"), }, { kubeReserved:   getResourceList("", "100Mi"), systemReserved: getResourceList("", "50Mi"), capacity:       getResourceList("10", ""), expected:       getResourceList("10", ""), }, } for idx, tc := range testCases { nc := NodeConfig{ NodeAllocatableConfig: NodeAllocatableConfig{ KubeReserved:   tc.kubeReserved, SystemReserved: tc.systemReserved, HardEvictionThresholds: []evictionapi.Threshold{ { Signal:   evictionapi.SignalMemoryAvailable, Operator: evictionapi.OpLessThan, Value:    tc.hardThreshold, }, }, }, } cm := &containerManagerImpl{ NodeConfig: nc, capacity:   tc.capacity, } for k, v := range cm.getNodeAllocatableAbsolute() { expected, exists := tc.expected[k] assert.True(t, exists) assert.Equal(t, expected.MilliValue(), v.MilliValue(), "test case %d failed for resource %q", idx+1, k) } } } func TestNodeAllocatableInputValidation(t *testing.T) { memoryEvictionThreshold := resource.MustParse("100Mi") highMemoryEvictionThreshold := resource.MustParse("2Gi") cpuMemTestCases := []struct { kubeReserved         v1.ResourceList systemReserved       v1.ResourceList capacity             v1.ResourceList hardThreshold        evictionapi.ThresholdValue invalidConfiguration bool }{ { kubeReserved:   getResourceList("100m", "100Mi"), systemReserved: getResourceList("50m", "50Mi"), capacity:       getResourceList("10", "10Gi"), }, { kubeReserved:   getResourceList("100m", "100Mi"), systemReserved: getResourceList("50m", "50Mi"), hardThreshold: evictionapi.ThresholdValue{ Quantity: &memoryEvictionThreshold, }, capacity: getResourceList("10", "10Gi"), }, { kubeReserved:   getResourceList("100m", "100Mi"), systemReserved: getResourceList("50m", "50Mi"), hardThreshold: evictionapi.ThresholdValue{ Percentage: 0.05, }, capacity: getResourceList("10", "10Gi"), }, { kubeReserved:   v1.ResourceList{}, systemReserved: v1.ResourceList{}, capacity:       getResourceList("10", "10Gi"), }, { kubeReserved:   getResourceList("", "100Mi"), systemReserved: getResourceList("50m", "50Mi"), capacity:       getResourceList("10", "10Gi"), }, { kubeReserved:   getResourceList("50m", "100Mi"), systemReserved: getResourceList("", "50Mi"), capacity:       getResourceList("10", "10Gi"), }, { kubeReserved:   getResourceList("", "100Mi"), systemReserved: getResourceList("", "50Mi"), capacity:       getResourceList("10", ""), }, { kubeReserved:   getResourceList("5", "10Gi"), systemReserved: getResourceList("5", "10Gi"), hardThreshold: evictionapi.ThresholdValue{ Quantity: &highMemoryEvictionThreshold, }, capacity:             getResourceList("10", "11Gi"), invalidConfiguration: true, }, } for _, tc := range cpuMemTestCases { nc := NodeConfig{ NodeAllocatableConfig: NodeAllocatableConfig{ KubeReserved:   tc.kubeReserved, SystemReserved: tc.systemReserved, HardEvictionThresholds: []evictionapi.Threshold{ { Signal:   evictionapi.SignalMemoryAvailable, Operator: evictionapi.OpLessThan, Value:    tc.hardThreshold, }, }, }, } cm := &containerManagerImpl{ NodeConfig: nc, capacity:   tc.capacity, } err := cm.validateNodeAllocatable() if err == nil && tc.invalidConfiguration { t.Fatalf("Expected invalid node allocatable configuration") } else if err != nil && !tc.invalidConfiguration { t.Fatalf("Expected valid node allocatable configuration: %v", err) } } ephemeralStorageEvictionThreshold := resource.MustParse("100Mi") ephemeralStorageTestCases := []struct { kubeReserved         v1.ResourceList capacity             v1.ResourceList hardThreshold        evictionapi.ThresholdValue invalidConfiguration bool }{ { kubeReserved: getEphemeralStorageResourceList("100Mi"), capacity:     getEphemeralStorageResourceList("500Mi"), }, { kubeReserved: getEphemeralStorageResourceList("20Gi"), hardThreshold: evictionapi.ThresholdValue{ Quantity: &ephemeralStorageEvictionThreshold, }, capacity:             getEphemeralStorageResourceList("20Gi"), invalidConfiguration: true, }, } for _, tc := range ephemeralStorageTestCases { nc := NodeConfig{ NodeAllocatableConfig: NodeAllocatableConfig{ KubeReserved: tc.kubeReserved, HardEvictionThresholds: []evictionapi.Threshold{ { Signal:   evictionapi.SignalNodeFsAvailable, Operator: evictionapi.OpLessThan, Value:    tc.hardThreshold, }, }, }, } cm := &containerManagerImpl{ NodeConfig: nc, capacity:   tc.capacity, } err := cm.validateNodeAllocatable() if err == nil && tc.invalidConfiguration { t.Fatalf("Expected invalid node allocatable configuration") } else if err != nil && !tc.invalidConfiguration { t.Fatalf("Expected valid node allocatable configuration: %v", err) } } } // getEphemeralStorageResourceList returns a ResourceList with the // specified ephemeral storage resource values func getEphemeralStorageResourceList(storage string) v1.ResourceList { res := v1.ResourceList{} if storage != "" { res[v1.ResourceEphemeralStorage] = resource.MustParse(storage) } return res } </ins>

<del> // +build linux /* Copyright 2017 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package cm import ( "testing" "github.com/stretchr/testify/assert" "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" evictionapi "k8s.io/kubernetes/pkg/kubelet/eviction/api" ) func TestNodeAllocatableReservationForScheduling(t *testing.T) { memoryEvictionThreshold := resource.MustParse("100Mi") cpuMemCases := []struct { kubeReserved   v1.ResourceList systemReserved v1.ResourceList expected       v1.ResourceList capacity       v1.ResourceList hardThreshold  evictionapi.ThresholdValue }{ { kubeReserved:   getResourceList("100m", "100Mi"), systemReserved: getResourceList("50m", "50Mi"), capacity:       getResourceList("10", "10Gi"), expected:       getResourceList("150m", "150Mi"), }, { kubeReserved:   getResourceList("100m", "100Mi"), systemReserved: getResourceList("50m", "50Mi"), hardThreshold: evictionapi.ThresholdValue{ Quantity: &memoryEvictionThreshold, }, capacity: getResourceList("10", "10Gi"), expected: getResourceList("150m", "250Mi"), }, { kubeReserved:   getResourceList("100m", "100Mi"), systemReserved: getResourceList("50m", "50Mi"), capacity:       getResourceList("10", "10Gi"), hardThreshold: evictionapi.ThresholdValue{ Percentage: 0.05, }, expected: getResourceList("150m", "694157320"), }, { kubeReserved:   v1.ResourceList{}, systemReserved: v1.ResourceList{}, capacity:       getResourceList("10", "10Gi"), expected:       getResourceList("", ""), }, { kubeReserved:   getResourceList("", "100Mi"), systemReserved: getResourceList("50m", "50Mi"), capacity:       getResourceList("10", "10Gi"), expected:       getResourceList("50m", "150Mi"), }, { kubeReserved:   getResourceList("50m", "100Mi"), systemReserved: getResourceList("", "50Mi"), capacity:       getResourceList("10", "10Gi"), expected:       getResourceList("50m", "150Mi"), }, { kubeReserved:   getResourceList("", "100Mi"), systemReserved: getResourceList("", "50Mi"), capacity:       getResourceList("10", ""), expected:       getResourceList("", "150Mi"), }, } for idx, tc := range cpuMemCases { nc := NodeConfig{ NodeAllocatableConfig: NodeAllocatableConfig{ KubeReserved:   tc.kubeReserved, SystemReserved: tc.systemReserved, HardEvictionThresholds: []evictionapi.Threshold{ { Signal:   evictionapi.SignalMemoryAvailable, Operator: evictionapi.OpLessThan, Value:    tc.hardThreshold, }, }, }, } cm := &containerManagerImpl{ NodeConfig: nc, capacity:   tc.capacity, } for k, v := range cm.GetNodeAllocatableReservation() { expected, exists := tc.expected[k] assert.True(t, exists, "test case %d expected resource %q", idx+1, k) assert.Equal(t, expected.MilliValue(), v.MilliValue(), "test case %d failed for resource %q", idx+1, k) } } ephemeralStorageEvictionThreshold := resource.MustParse("100Mi") ephemeralStorageTestCases := []struct { kubeReserved  v1.ResourceList expected      v1.ResourceList capacity      v1.ResourceList hardThreshold evictionapi.ThresholdValue }{ { kubeReserved: getEphemeralStorageResourceList("100Mi"), capacity:     getEphemeralStorageResourceList("10Gi"), expected:     getEphemeralStorageResourceList("100Mi"), }, { kubeReserved: getEphemeralStorageResourceList("100Mi"), hardThreshold: evictionapi.ThresholdValue{ Quantity: &ephemeralStorageEvictionThreshold, }, capacity: getEphemeralStorageResourceList("10Gi"), expected: getEphemeralStorageResourceList("200Mi"), }, { kubeReserved: getEphemeralStorageResourceList("150Mi"), capacity:     getEphemeralStorageResourceList("10Gi"), hardThreshold: evictionapi.ThresholdValue{ Percentage: 0.05, }, expected: getEphemeralStorageResourceList("694157320"), }, { kubeReserved: v1.ResourceList{}, capacity:     getEphemeralStorageResourceList("10Gi"), expected:     getEphemeralStorageResourceList(""), }, } for idx, tc := range ephemeralStorageTestCases { nc := NodeConfig{ NodeAllocatableConfig: NodeAllocatableConfig{ KubeReserved: tc.kubeReserved, HardEvictionThresholds: []evictionapi.Threshold{ { Signal:   evictionapi.SignalNodeFsAvailable, Operator: evictionapi.OpLessThan, Value:    tc.hardThreshold, }, }, }, } cm := &containerManagerImpl{ NodeConfig: nc, capacity:   tc.capacity, } for k, v := range cm.GetNodeAllocatableReservation() { expected, exists := tc.expected[k] assert.True(t, exists, "test case %d expected resource %q", idx+1, k) assert.Equal(t, expected.MilliValue(), v.MilliValue(), "test case %d failed for resource %q", idx+1, k) } } } func TestNodeAllocatableForEnforcement(t *testing.T) { memoryEvictionThreshold := resource.MustParse("100Mi") testCases := []struct { kubeReserved   v1.ResourceList systemReserved v1.ResourceList capacity       v1.ResourceList expected       v1.ResourceList hardThreshold  evictionapi.ThresholdValue }{ { kubeReserved:   getResourceList("100m", "100Mi"), systemReserved: getResourceList("50m", "50Mi"), capacity:       getResourceList("10", "10Gi"), expected:       getResourceList("9850m", "10090Mi"), }, { kubeReserved:   getResourceList("100m", "100Mi"), systemReserved: getResourceList("50m", "50Mi"), hardThreshold: evictionapi.ThresholdValue{ Quantity: &memoryEvictionThreshold, }, capacity: getResourceList("10", "10Gi"), expected: getResourceList("9850m", "10090Mi"), }, { kubeReserved:   getResourceList("100m", "100Mi"), systemReserved: getResourceList("50m", "50Mi"), hardThreshold: evictionapi.ThresholdValue{ Percentage: 0.05, }, capacity: getResourceList("10", "10Gi"), expected: getResourceList("9850m", "10090Mi"), }, { kubeReserved:   v1.ResourceList{}, systemReserved: v1.ResourceList{}, capacity:       getResourceList("10", "10Gi"), expected:       getResourceList("10", "10Gi"), }, { kubeReserved:   getResourceList("", "100Mi"), systemReserved: getResourceList("50m", "50Mi"), capacity:       getResourceList("10", "10Gi"), expected:       getResourceList("9950m", "10090Mi"), }, { kubeReserved:   getResourceList("50m", "100Mi"), systemReserved: getResourceList("", "50Mi"), capacity:       getResourceList("10", "10Gi"), expected:       getResourceList("9950m", "10090Mi"), }, { kubeReserved:   getResourceList("", "100Mi"), systemReserved: getResourceList("", "50Mi"), capacity:       getResourceList("10", ""), expected:       getResourceList("10", ""), }, } for idx, tc := range testCases { nc := NodeConfig{ NodeAllocatableConfig: NodeAllocatableConfig{ KubeReserved:   tc.kubeReserved, SystemReserved: tc.systemReserved, HardEvictionThresholds: []evictionapi.Threshold{ { Signal:   evictionapi.SignalMemoryAvailable, Operator: evictionapi.OpLessThan, Value:    tc.hardThreshold, }, }, }, } cm := &containerManagerImpl{ NodeConfig: nc, capacity:   tc.capacity, } for k, v := range cm.getNodeAllocatableAbsolute() { expected, exists := tc.expected[k] assert.True(t, exists) assert.Equal(t, expected.MilliValue(), v.MilliValue(), "test case %d failed for resource %q", idx+1, k) } } } func TestNodeAllocatableInputValidation(t *testing.T) { memoryEvictionThreshold := resource.MustParse("100Mi") highMemoryEvictionThreshold := resource.MustParse("2Gi") cpuMemTestCases := []struct { kubeReserved         v1.ResourceList systemReserved       v1.ResourceList capacity             v1.ResourceList hardThreshold        evictionapi.ThresholdValue invalidConfiguration bool }{ { kubeReserved:   getResourceList("100m", "100Mi"), systemReserved: getResourceList("50m", "50Mi"), capacity:       getResourceList("10", "10Gi"), }, { kubeReserved:   getResourceList("100m", "100Mi"), systemReserved: getResourceList("50m", "50Mi"), hardThreshold: evictionapi.ThresholdValue{ Quantity: &memoryEvictionThreshold, }, capacity: getResourceList("10", "10Gi"), }, { kubeReserved:   getResourceList("100m", "100Mi"), systemReserved: getResourceList("50m", "50Mi"), hardThreshold: evictionapi.ThresholdValue{ Percentage: 0.05, }, capacity: getResourceList("10", "10Gi"), }, { kubeReserved:   v1.ResourceList{}, systemReserved: v1.ResourceList{}, capacity:       getResourceList("10", "10Gi"), }, { kubeReserved:   getResourceList("", "100Mi"), systemReserved: getResourceList("50m", "50Mi"), capacity:       getResourceList("10", "10Gi"), }, { kubeReserved:   getResourceList("50m", "100Mi"), systemReserved: getResourceList("", "50Mi"), capacity:       getResourceList("10", "10Gi"), }, { kubeReserved:   getResourceList("", "100Mi"), systemReserved: getResourceList("", "50Mi"), capacity:       getResourceList("10", ""), }, { kubeReserved:   getResourceList("5", "10Gi"), systemReserved: getResourceList("5", "10Gi"), hardThreshold: evictionapi.ThresholdValue{ Quantity: &highMemoryEvictionThreshold, }, capacity:             getResourceList("10", "11Gi"), invalidConfiguration: true, }, } for _, tc := range cpuMemTestCases { nc := NodeConfig{ NodeAllocatableConfig: NodeAllocatableConfig{ KubeReserved:   tc.kubeReserved, SystemReserved: tc.systemReserved, HardEvictionThresholds: []evictionapi.Threshold{ { Signal:   evictionapi.SignalMemoryAvailable, Operator: evictionapi.OpLessThan, Value:    tc.hardThreshold, }, }, }, } cm := &containerManagerImpl{ NodeConfig: nc, capacity:   tc.capacity, } err := cm.validateNodeAllocatable() if err == nil && tc.invalidConfiguration { t.Fatalf("Expected invalid node allocatable configuration") } else if err != nil && !tc.invalidConfiguration { t.Fatalf("Expected valid node allocatable configuration: %v", err) } } ephemeralStorageEvictionThreshold := resource.MustParse("100Mi") ephemeralStorageTestCases := []struct { kubeReserved         v1.ResourceList capacity             v1.ResourceList hardThreshold        evictionapi.ThresholdValue invalidConfiguration bool }{ { kubeReserved: getEphemeralStorageResourceList("100Mi"), capacity:     getEphemeralStorageResourceList("500Mi"), }, { kubeReserved: getEphemeralStorageResourceList("20Gi"), hardThreshold: evictionapi.ThresholdValue{ Quantity: &ephemeralStorageEvictionThreshold, }, capacity:             getEphemeralStorageResourceList("20Gi"), invalidConfiguration: true, }, } for _, tc := range ephemeralStorageTestCases { nc := NodeConfig{ NodeAllocatableConfig: NodeAllocatableConfig{ KubeReserved: tc.kubeReserved, HardEvictionThresholds: []evictionapi.Threshold{ { Signal:   evictionapi.SignalNodeFsAvailable, Operator: evictionapi.OpLessThan, Value:    tc.hardThreshold, }, }, }, } cm := &containerManagerImpl{ NodeConfig: nc, capacity:   tc.capacity, } err := cm.validateNodeAllocatable() if err == nil && tc.invalidConfiguration { t.Fatalf("Expected invalid node allocatable configuration") } else if err != nil && !tc.invalidConfiguration { t.Fatalf("Expected valid node allocatable configuration: %v", err) } } } // getEphemeralStorageResourceList returns a ResourceList with the // specified ephemeral storage resource values func getEphemeralStorageResourceList(storage string) v1.ResourceList { res := v1.ResourceList{} if storage != "" { res[v1.ResourceEphemeralStorage] = resource.MustParse(storage) } return res } </del>