func (r *MPIJobReconciler) GetReconcileOrders() []v1.ReplicaType {
	return []v1.ReplicaType{
		training.MPIReplicaTypeWorker,
		training.MPIReplicaTypeLauncher,
	}
}
Get replicas reconcile orders so that replica type with higher priority can be created earlier.
package mpi
import (
	"context"
	stderrors "errors"
	"fmt"
	"strings"
	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/api/resource"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/client-go/tools/record"
	ctrl "sigs.k8s.io/controller-runtime"
	"sigs.k8s.io/controller-runtime/pkg/client"
	"sigs.k8s.io/controller-runtime/pkg/controller"
	"sigs.k8s.io/controller-runtime/pkg/handler"
	"sigs.k8s.io/controller-runtime/pkg/predicate"
	"sigs.k8s.io/controller-runtime/pkg/reconcile"
	logf "sigs.k8s.io/controller-runtime/pkg/runtime/log"
	"sigs.k8s.io/controller-runtime/pkg/source"
	training "github.com/alibaba/kubedl/apis/training/v1alpha1"
	"github.com/alibaba/kubedl/cmd/options"
	"github.com/alibaba/kubedl/pkg/gang_schedule/registry"
	"github.com/alibaba/kubedl/pkg/job_controller"
	v1 "github.com/alibaba/kubedl/pkg/job_controller/api/v1"
	"github.com/alibaba/kubedl/pkg/metrics"
	"github.com/alibaba/kubedl/pkg/util"
	"github.com/spf13/pflag"
)
func init() {
	pflag.StringVar(&kubectlDeliveryImage, "kubectl-delivery-image", defaultKubectlDeliveryImage, "utility image to delivery kubectl binary")
	pflag.BoolVar(&launcherRunsWorkload, "launcher-runs-workloads", true, "Set launcher run the workload when launcher has GPU")
}
const (
	controllerName              = "MPIController"
	defaultKubectlDeliveryImage = "kubedl/kubectl-delivery:latest"
	initContainerCpu            = "100m"
	initContainerMem            = "128Mi"
)
var (
	log = logf.Log.WithName("mpi-controller")
	kubectlDeliveryImage string
	launcherRunsWorkload bool
)
func NewReconciler(mgr ctrl.Manager, config options.JobControllerConfiguration) *MPIJobReconciler {
	r := &MPIJobReconciler{
		Client: mgr.GetClient(),
		scheme: mgr.GetScheme(),
	}
	r.recorder = mgr.GetEventRecorderFor(r.ControllerName())
	r.ctrl = job_controller.NewJobController(r.Client, r, config, r.recorder, metrics.NewJobMetrics(training.MPIJobKind, r.Client))
	if r.ctrl.Config.EnableGangScheduling {
		r.ctrl.GangScheduler = registry.Get(r.ctrl.Config.GangSchedulerName)
	}
	return r
}
var _ reconcile.Reconciler = &MPIJobReconciler{}
var _ v1.ControllerInterface = &MPIJobReconciler{}
type MPIJobReconciler struct {
	client.Client
	scheme   *runtime.Scheme
	recorder record.EventRecorder
	ctrl     job_controller.JobController
}
func (r *MPIJobReconciler) Reconcile(req ctrl.Request) (ctrl.Result, error) {
	var (
		result ctrl.Result
		err    error
	)
	sharedMPIJob := &training.MPIJob{}
	err = util.GetObjectByPassCache(r.Client, req.NamespacedName, sharedMPIJob)
	if err != nil {
		if errors.IsNotFound(err) {
			log.Info("try to get job but it has been deleted", "key", req.String())
			r.ctrl.Metrics.DeletedInc()
			return reconcile.Result{}, nil
		}
		return reconcile.Result{}, err
	}
	mpiJob := sharedMPIJob.DeepCopy()
	needSync := r.ctrl.SatisfyExpectations(mpiJob, mpiJob.Spec.MPIReplicaSpecs)
	if !needSync || mpiJob.DeletionTimestamp != nil {
		log.Info("reconcile cancelled, job does not need to do reconcile or has been deleted",
			"job namespace", mpiJob.Namespace, "job name", mpiJob.Name,
			"need sync", needSync, "deleted", mpiJob.DeletionTimestamp != nil)
		return reconcile.Result{}, nil
	}
	if err = LegacyMPIJobToV1MPIJob(mpiJob); err != nil {
		log.Error(err, "handle legacy fields of mpi job failed")
		return result, err
	}
	r.scheme.Default(mpiJob)
	result, err = r.ctrl.ReconcileJobs(mpiJob, mpiJob.Spec.MPIReplicaSpecs, mpiJob.Status, &mpiJob.Spec.RunPolicy, nil)
	if err != nil {
		log.Error(err, "mpi job reconcile failed")
		return result, err
	}
	return result, err
}
func (r *MPIJobReconciler) SetupWithManager(mgr ctrl.Manager) error {
	c, err := controller.New(r.ControllerName(), mgr, controller.Options{
		Reconciler:              r,
		MaxConcurrentReconciles: options.CtrlConfig.MaxConcurrentReconciles,
	})
	if err != nil {
		return err
	}
	if err = c.Watch(&source.Kind{Type: &training.MPIJob{}}, &handler.EnqueueRequestForObject{}, predicate.Funcs{
		CreateFunc: onOwnerCreateFunc(r),
	}); err != nil {
		return err
	}
	if err = c.Watch(&source.Kind{Type: &corev1.Pod{}}, &handler.EnqueueRequestForOwner{
		OwnerType:    &training.MPIJob{},
		IsController: true,
	}, predicate.Funcs{
		CreateFunc: r.ctrl.OnPodCreateFunc,
		UpdateFunc: r.ctrl.OnPodUpdateFunc,
		DeleteFunc: r.ctrl.OnPodDeleteFunc,
	}); err != nil {
		return err
	}
	if err = c.Watch(&source.Kind{Type: &corev1.ConfigMap{}}, &handler.EnqueueRequestForOwner{
		OwnerType:    &training.MPIJob{},
		IsController: true,
	}); err != nil {
		return err
	}
	return nil
}
func (r *MPIJobReconciler) ControllerName() string {
	return controllerName
}
func (r *MPIJobReconciler) GetAPIGroupVersionKind() schema.GroupVersionKind {
	return training.SchemeGroupVersion.WithKind(training.MPIJobKind)
}
func (r *MPIJobReconciler) GetAPIGroupVersion() schema.GroupVersion {
	return training.SchemeGroupVersion
}
func (r *MPIJobReconciler) GetGroupNameLabelValue() string {
	return training.SchemeGroupVersion.Group
}
func (r *MPIJobReconciler) SetClusterSpec(ctx context.Context, job interface{}, podTemplateSpec *corev1.PodTemplateSpec, rt, index string) error {
	mpiJob, ok := job.(*training.MPIJob)
	if !ok {
		return fmt.Errorf("%+v is not a type of MPIJob", job)
	}
	workerReplicas := int32(0)
	if workerSpec := mpiJob.Spec.MPIReplicaSpecs[training.MPIReplicaTypeWorker]; workerSpec != nil && workerSpec.Replicas != nil {
		workerReplicas = *workerSpec.Replicas
	}
	if launcherSpec, ok := mpiJob.Spec.MPIReplicaSpecs[training.MPIReplicaTypeLauncher]; ok && launcherSpec != nil {
		log.Info("launcher of MPIJob: ", mpiJob.Namespace+"/"+mpiJob.Name, " generate job config.")
		_, err := r.getOrCreateJobConfig(mpiJob, workerReplicas, launcherRunsWorkload)
		if err != nil {
			return err
		}
	}
	switch rt {
	case strings.ToLower(string(training.MPIReplicaTypeWorker)):
		r.setupMPIWorker(mpiJob, podTemplateSpec)
	case strings.ToLower(string(training.MPIReplicaTypeLauncher)):
		r.setupMPILauncher(mpiJob, podTemplateSpec)
	}
	return nil
}
func (r *MPIJobReconciler) GetDefaultContainerName() string {
	return training.MPIJobDefaultContainerName
}
func (r *MPIJobReconciler) GetDefaultContainerPortName() string {
	return training.MPIJobDefaultPortName
}
func (r *MPIJobReconciler) GetDefaultContainerPortNumber() int32 {
	return training.MPIJobDefaultPort
}