Change:
/sig scheduling What happened: reverted an old behavior to ensure Pod information is not lost when Nodes are recreated without its Pods getting recreated. A ghost Pod is kep in The problem is that we rely in this map for determining deleted Nodes when updating the snapshot in . The good thing is that the nodeinfo's are cloned into the snapshot, so we shouldn't have nil pointer accesses. But we might lose a few cycles trying to schedule on nodes that are deleted How to reproduce it (as minimally and precisely as possible): a node with a few pods. node without deleting pods. Result: The snapshot still has the node. Environment: Kubernetes version (use ): 1.18+
/assign
It actually produces the error: which would cause failed scheduling cycles (the map is not recreated, only the lists).

Discussion:
// UpdateSnapshot takes a snapshot of cached NodeInfo map. This is called at // beginning of every scheduling cycle. <ins> // The snapshot only includes Nodes that are not deleted at the time this function is called. // nodeinfo.Node() is guaranteed to be not nil for all the nodes in the snapshot. </ins> // This function tracks generation number of NodeInfo and updates only the // entries of an existing snapshot that have changed after the snapshot was taken. func (cache *schedulerCache) UpdateSnapshot(nodeSnapshot *Snapshot) error {

nodeSnapshot.generation = cache.headNode.info.Generation } <del> if len(nodeSnapshot.nodeInfoMap) > len(cache.nodes) { </del> <ins> // Comparing to pods in nodeTree. // Deleted nodes get removed from the tree, but they might remain in the nodes map // if they still have non-deleted Pods. if len(nodeSnapshot.nodeInfoMap) > cache.nodeTree.numNodes { </ins> cache.removeDeletedNodesFromSnapshot(nodeSnapshot) updateAllLists = true }

// If certain nodes were deleted after the last snapshot was taken, we should remove them from the snapshot. func (cache *schedulerCache) removeDeletedNodesFromSnapshot(snapshot *Snapshot) { <del> toDelete := len(snapshot.nodeInfoMap) - len(cache.nodes) </del> <ins> toDelete := len(snapshot.nodeInfoMap) - cache.nodeTree.numNodes </ins> for name := range snapshot.nodeInfoMap { if toDelete <= 0 { break } <del> if _, ok := cache.nodes[name]; !ok { </del> <ins> if n, ok := cache.nodes[name]; !ok || n.info.Node() == nil { </ins> delete(snapshot.nodeInfoMap, name) toDelete-- }

var cache *schedulerCache var snapshot *Snapshot <del> type operation = func() </del> <ins> type operation = func(t *testing.T) </ins> addNode := func(i int) operation { <del> return func() { </del> <ins> return func(t *testing.T) { </ins> if err := cache.AddNode(nodes[i]); err != nil { t.Error(err) } } } removeNode := func(i int) operation { <del> return func() { </del> <ins> return func(t *testing.T) { </ins> if err := cache.RemoveNode(nodes[i]); err != nil { t.Error(err) } } } updateNode := func(i int) operation { <del> return func() { </del> <ins> return func(t *testing.T) { </ins> if err := cache.UpdateNode(nodes[i], updatedNodes[i]); err != nil { t.Error(err) } } } addPod := func(i int) operation { <del> return func() { </del> <ins> return func(t *testing.T) { </ins> if err := cache.AddPod(pods[i]); err != nil { t.Error(err) } } } addPodWithAffinity := func(i int) operation { <del> return func() { </del> <ins> return func(t *testing.T) { </ins> if err := cache.AddPod(podsWithAffinity[i]); err != nil { t.Error(err) } } } removePod := func(i int) operation { <del> return func() { </del> <ins> return func(t *testing.T) { </ins> if err := cache.RemovePod(pods[i]); err != nil { t.Error(err) } } } removePodWithAffinity := func(i int) operation { <del> return func() { </del> <ins> return func(t *testing.T) { </ins> if err := cache.RemovePod(podsWithAffinity[i]); err != nil { t.Error(err) } } } updatePod := func(i int) operation { <del> return func() { </del> <ins> return func(t *testing.T) { </ins> if err := cache.UpdatePod(pods[i], updatedPods[i]); err != nil { t.Error(err) } } } updateSnapshot := func() operation { <del> return func() { </del> <ins> return func(t *testing.T) { </ins> cache.UpdateSnapshot(snapshot) if err := compareCacheWithNodeInfoSnapshot(t, cache, snapshot); err != nil { t.Error(err)

{ name: "Remove node before its pods", operations: []operation{ <del> addNode(0), addNode(1), addPod(1), addPod(11), removeNode(1), updatePod(1), updatePod(11), removePod(1), removePod(11), </del> <ins> addNode(0), addNode(1), addPod(1), addPod(11), updateSnapshot(), removeNode(1), updateSnapshot(), updatePod(1), updatePod(11), removePod(1), removePod(11), </ins> }, expected: []*v1.Node{nodes[0]}, },

snapshot = NewEmptySnapshot() for _, op := range test.operations { <del> op() </del> <ins> op(t) </ins> } if len(test.expected) != len(cache.nodes) {

func compareCacheWithNodeInfoSnapshot(t *testing.T, cache *schedulerCache, snapshot *Snapshot) error { // Compare the map. <del> if len(snapshot.nodeInfoMap) != len(cache.nodes) { return fmt.Errorf("unexpected number of nodes in the snapshot. Expected: %v, got: %v", len(cache.nodes), len(snapshot.nodeInfoMap)) </del> <ins> if len(snapshot.nodeInfoMap) != cache.nodeTree.numNodes { return fmt.Errorf("unexpected number of nodes in the snapshot. Expected: %v, got: %v", cache.nodeTree.numNodes, len(snapshot.nodeInfoMap)) </ins> } for name, ni := range cache.nodes { <del> if !reflect.DeepEqual(snapshot.nodeInfoMap[name], ni.info) { return fmt.Errorf("unexpected node info for node %q. Expected: %v, got: %v", name, ni.info, snapshot.nodeInfoMap[name]) </del> <ins> want := ni.info if want.Node() == nil { want = nil } if !reflect.DeepEqual(snapshot.nodeInfoMap[name], want) { return fmt.Errorf("unexpected node info for node %q.Expected:n%v, got:n%v", name, ni.info, snapshot.nodeInfoMap[name]) </ins> } } // Compare the lists. <del> if len(snapshot.nodeInfoList) != len(cache.nodes) { return fmt.Errorf("unexpected number of nodes in NodeInfoList. Expected: %v, got: %v", len(cache.nodes), len(snapshot.nodeInfoList)) </del> <ins> if len(snapshot.nodeInfoList) != cache.nodeTree.numNodes { return fmt.Errorf("unexpected number of nodes in NodeInfoList. Expected: %v, got: %v", cache.nodeTree.numNodes, len(snapshot.nodeInfoList)) </ins> } expectedNodeInfoList := make([]*framework.NodeInfo, 0, cache.nodeTree.numNodes)

package cache import ( <del> "k8s.io/api/core/v1" </del> <ins> v1 "k8s.io/api/core/v1" </ins> framework "k8s.io/kubernetes/pkg/scheduler/framework/v1alpha1" )

// UpdateSnapshot updates the passed infoSnapshot to the current contents of Cache. // The node info contains aggregated information of pods scheduled (including assumed to be) // on this node. <ins> // The snapshot only includes Nodes that are not deleted at the time this function is called. // nodeinfo.Node() is guaranteed to be not nil for all the nodes in the snapshot. </ins> UpdateSnapshot(nodeSnapshot *Snapshot) error // Dump produces a dump of the current cache.