Repository: volcano-sh/volcano (Go)

Issue:
Plugin `predicates` won't remove pod from node during the rollback of scheduling transaction
<!-- This form is for bug reports ONLY!

If you're looking for a help then check our [Slack Channel](https://cloud-native.slack.com/messages/volcano) or have a look at our [dev mailing](https://groups.google.com/forum/#!forum/volcano-sh) 
-->

**What happened**:
+ `Predicates.DeallocateFunc` won't remove pod from node(which defined in `predicatesPlugin.OnSessionOpen.nodeMap`) during the rollback of scheduling transaction(or say statement discarded).
+ Instead, it leaves an error log: "predicates, update pod xxx allocate from NOT EXIST node []"

**What you expected to happen**:
+ Obviously it should successfully remove the pod assigned on that node when scheduling transaction is rollbacked.

**How to reproduce it (as minimally and precisely as possible)**:
+ Turn on plugin `predicates` & `gang`.
+ Scheduling a podgroup with 2 pods, and one of the pod(let's say pod1) can be successfully scheduled(let's say on node1) and the other is unschedulable.
+ Thus the entire podgroup is gang unschedulable, and the scheduling transaction should be rollbacked.
+ This bug occurs there when `node.RemoveTask(task)` is called in func statement.unallocate.
   - It set node name of the task to empty(which i dont know why) and hence the node can not be found in `Predicates.DeallocateFunc` due to the empty nodeName.
     ```go
     ti.NodeName = "" // remove nodename from taskinfo
     ```

**Anything else we need to know?**:
+ It seams the code i cited above was introduced in PR #1517 . Why changing that and what's this all about?



Discussion:
This can be removed as there is `task.NodeName = ""` already in unpipeline, unallocate. 
> This can be removed as there is `task.NodeName = ""` already in unpipeline, unallocate.

@lowang-bh fixed by #1716 , PTAL



Patch:
diff --git a/pkg/scheduler/api/node_info.go b/pkg/scheduler/api/node_info.go
index bedb498ff4..59edb3e8fa 100644
--- a/pkg/scheduler/api/node_info.go
+++ b/pkg/scheduler/api/node_info.go
@@ -418,7 +418,6 @@ func (ni *NodeInfo) RemoveTask(ti *TaskInfo) error {
 		}
 	}
 
-	ti.NodeName = "" // remove nodename from taskinfo
 	delete(ni.Tasks, key)
 
 	return nil


Test patch:
diff --git a/pkg/scheduler/plugins/predicates/predicates_test.go b/pkg/scheduler/plugins/predicates/predicates_test.go
new file mode 100644
index 0000000000..88eeb90731
--- /dev/null
+++ b/pkg/scheduler/plugins/predicates/predicates_test.go
@@ -0,0 +1,207 @@
+package predicates
+
+import (
+	"reflect"
+	"testing"
+	"time"
+
+	"volcano.sh/volcano/cmd/scheduler/app/options"
+	"volcano.sh/volcano/pkg/scheduler/actions/allocate"
+	"volcano.sh/volcano/pkg/scheduler/api"
+	"volcano.sh/volcano/pkg/scheduler/cache"
+	"volcano.sh/volcano/pkg/scheduler/conf"
+	"volcano.sh/volcano/pkg/scheduler/framework"
+	"volcano.sh/volcano/pkg/scheduler/plugins/gang"
+	"volcano.sh/volcano/pkg/scheduler/plugins/priority"
+	"volcano.sh/volcano/pkg/scheduler/util"
+
+	apiv1 "k8s.io/api/core/v1"
+	"k8s.io/api/scheduling/v1beta1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/tools/record"
+	schedulingv1 "volcano.sh/apis/pkg/apis/scheduling/v1beta1"
+)
+
+func getWorkerAffinity() *apiv1.Affinity {
+	return &apiv1.Affinity{
+		PodAntiAffinity: &apiv1.PodAntiAffinity{
+			RequiredDuringSchedulingIgnoredDuringExecution: []apiv1.PodAffinityTerm{
+				{
+					LabelSelector: &metav1.LabelSelector{
+						MatchExpressions: []metav1.LabelSelectorRequirement{
+							{
+								Key:      "role",
+								Operator: "In",
+								Values:   []string{"worker"},
+							},
+						},
+					},
+					TopologyKey: "kubernetes.io/hostname",
+				},
+			},
+		},
+	}
+}
+
+func TestEventHandler(t *testing.T) {
+	framework.RegisterPluginBuilder(PluginName, New)
+	framework.RegisterPluginBuilder(gang.PluginName, gang.New)
+	framework.RegisterPluginBuilder(priority.PluginName, priority.New)
+	options.ServerOpts = options.NewServerOption()
+	defer framework.CleanupPluginBuilders()
+
+	// pending pods
+	w1 := util.BuildPod("ns1", "worker-1", "", apiv1.PodPending, util.BuildResourceList("3", "3k"), "pg1", map[string]string{"role": "worker"}, map[string]string{"selector": "worker"})
+	w2 := util.BuildPod("ns1", "worker-2", "", apiv1.PodPending, util.BuildResourceList("5", "5k"), "pg1", map[string]string{"role": "worker"}, map[string]string{})
+	w3 := util.BuildPod("ns1", "worker-3", "", apiv1.PodPending, util.BuildResourceList("4", "4k"), "pg2", map[string]string{"role": "worker"}, map[string]string{})
+	w1.Spec.Affinity = getWorkerAffinity()
+	w2.Spec.Affinity = getWorkerAffinity()
+	w3.Spec.Affinity = getWorkerAffinity()
+
+	// nodes
+	n1 := util.BuildNode("node1", util.BuildResourceList("4", "4k"), map[string]string{"selector": "worker"})
+	n2 := util.BuildNode("node2", util.BuildResourceList("3", "3k"), map[string]string{})
+	n1.Status.Allocatable["pods"] = resource.MustParse("15")
+	n2.Status.Allocatable["pods"] = resource.MustParse("15")
+	n1.Labels["kubernetes.io/hostname"] = "node1"
+	n2.Labels["kubernetes.io/hostname"] = "node2"
+
+	// priority
+	p1 := &v1beta1.PriorityClass{ObjectMeta: metav1.ObjectMeta{Name: "p1"}, Value: 1}
+	p2 := &v1beta1.PriorityClass{ObjectMeta: metav1.ObjectMeta{Name: "p2"}, Value: 2}
+	// podgroup
+	pg1 := &schedulingv1.PodGroup{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: "ns1",
+			Name:      "pg1",
+		},
+		Spec: schedulingv1.PodGroupSpec{
+			Queue:             "q1",
+			MinMember:         int32(2),
+			PriorityClassName: p2.Name,
+		},
+	}
+	pg2 := &schedulingv1.PodGroup{
+		ObjectMeta: metav1.ObjectMeta{
+			Namespace: "ns1",
+			Name:      "pg2",
+		},
+		Spec: schedulingv1.PodGroupSpec{
+			Queue:             "q1",
+			MinMember:         int32(1),
+			PriorityClassName: p1.Name,
+		},
+	}
+	// queue
+	queue1 := &schedulingv1.Queue{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: "q1",
+		},
+	}
+
+	// tests
+	tests := []struct {
+		name     string
+		pods     []*apiv1.Pod
+		nodes    []*apiv1.Node
+		pcs      []*v1beta1.PriorityClass
+		pgs      []*schedulingv1.PodGroup
+		expected map[string]string
+	}{
+		{
+			name:  "pod-deallocate",
+			pods:  []*apiv1.Pod{w1, w2, w3},
+			nodes: []*apiv1.Node{n1, n2},
+			pcs:   []*v1beta1.PriorityClass{p1, p2},
+			pgs:   []*schedulingv1.PodGroup{pg1, pg2},
+			expected: map[string]string{ // podKey -> node
+				"ns1/worker-3": "node1",
+			},
+		},
+	}
+
+	for _, test := range tests {
+		// initialize schedulerCache
+		binder := &util.FakeBinder{
+			Binds:   map[string]string{},
+			Channel: make(chan string),
+		}
+		recorder := record.NewFakeRecorder(100)
+		go func() {
+			for {
+				event := <-recorder.Events
+				t.Logf("%s: [Event] %s", test.name, event)
+			}
+		}()
+		schedulerCache := &cache.SchedulerCache{
+			Nodes:           make(map[string]*api.NodeInfo),
+			Jobs:            make(map[api.JobID]*api.JobInfo),
+			PriorityClasses: make(map[string]*v1beta1.PriorityClass),
+			Queues:          make(map[api.QueueID]*api.QueueInfo),
+			Binder:          binder,
+			StatusUpdater:   &util.FakeStatusUpdater{},
+			VolumeBinder:    &util.FakeVolumeBinder{},
+			Recorder:        recorder,
+		}
+		for _, node := range test.nodes {
+			schedulerCache.AddNode(node)
+		}
+		for _, pod := range test.pods {
+			schedulerCache.AddPod(pod)
+		}
+		for _, pc := range test.pcs {
+			schedulerCache.PriorityClasses[pc.Name] = pc
+		}
+		for _, pg := range test.pgs {
+			pg.Status = schedulingv1.PodGroupStatus{
+				Phase: schedulingv1.PodGroupInqueue,
+			}
+			schedulerCache.AddPodGroupV1beta1(pg)
+		}
+		schedulerCache.AddQueueV1beta1(queue1)
+		// session
+		trueValue := true
+		ssn := framework.OpenSession(schedulerCache, []conf.Tier{
+			{
+				Plugins: []conf.PluginOption{
+					{
+						Name:             PluginName,
+						EnabledPredicate: &trueValue,
+					},
+					{
+						Name:                gang.PluginName,
+						EnabledJobReady:     &trueValue,
+						EnabledJobPipelined: &trueValue,
+					},
+					{
+						Name:            priority.PluginName,
+						EnabledJobOrder: &trueValue,
+					},
+				},
+			},
+		}, nil)
+		ssn.AddUnderusedResourceFn(PluginName, func(info *api.QueueInfo) api.ResourceNameList {
+			// to prevent queue from overusing
+			return api.ResourceNameList{apiv1.ResourceCPU, apiv1.ResourceMemory}
+		})
+		// allocate
+		allocator := allocate.New()
+		allocator.Execute(ssn)
+		framework.CloseSession(ssn)
+
+		// assert
+		for i := 0; i < len(test.expected); i++ {
+			select {
+			case <-binder.Channel:
+			case <-time.After(1 * time.Second):
+				t.Errorf("Failed to get binding request.")
+			}
+		}
+
+		t.Logf("expected: %#v, got: %#v", test.expected, binder.Binds)
+		if !reflect.DeepEqual(test.expected, binder.Binds) {
+			t.Errorf("expected: %v, got %v ", test.expected, binder.Binds)
+		}
+	}
+}