func generateTmpfsSupport(
	cr *kdv1.KubeDirectorCluster,
) ([]v1.VolumeMount, []v1.Volume) {
	volumeMounts := []v1.VolumeMount{
		v1.VolumeMount{
			Name:      "tmpfs-tmp",
			MountPath: "/tmp",
		},
		v1.VolumeMount{
			Name:      "tmpfs-run",
			MountPath: "/run",
		},
		v1.VolumeMount{
			Name:      "tmpfs-run-lock",
			MountPath: "/run/lock",
		},
	}
	maxTmpSize, _ := resource.ParseQuantity(tmpFSVolSize)
	volumes := []v1.Volume{
		v1.Volume{
			Name: "tmpfs-tmp",
			VolumeSource: v1.VolumeSource{
				EmptyDir: &v1.EmptyDirVolumeSource{
					Medium:    "Memory",
					SizeLimit: &maxTmpSize,
				},
			},
		},
		v1.Volume{
			Name: "tmpfs-run",
			VolumeSource: v1.VolumeSource{
				EmptyDir: &v1.EmptyDirVolumeSource{
					Medium:    "Memory",
					SizeLimit: &maxTmpSize,
				},
			},
		},
		v1.Volume{
			Name: "tmpfs-run-lock",
			VolumeSource: v1.VolumeSource{
				EmptyDir: &v1.EmptyDirVolumeSource{
					Medium:    "Memory",
					SizeLimit: &maxTmpSize,
				},
			},
		},
	}
	return volumeMounts, volumes
}
generateTmpfsSupport creates the volume and mount specs necessary for
backing an app container's /tmp and /run directories with a ramdisk. Limit
the size of the ramdisk to tmpFsVolSize.
package executor
import (
	"context"
	"path/filepath"
	"strconv"
	"strings"
	"github.com/go-logr/logr"
	"k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/types"
	"github.com/bluek8s/kubedirector/pkg/apis/kubedirector/v1beta1"
	kdv1 "github.com/bluek8s/kubedirector/pkg/apis/kubedirector/v1beta1"
	"github.com/bluek8s/kubedirector/pkg/catalog"
	"github.com/bluek8s/kubedirector/pkg/shared"
	appsv1 "k8s.io/api/apps/v1"
	v1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
var defaultMountFolders = []string{"/etc"}
var appConfigDefaultMountFolders = []string{"/etc", "/opt", "/usr"}
func CreateStatefulSet(
	reqLogger logr.Logger,
	cr *kdv1.KubeDirectorCluster,
	nativeSystemdSupport bool,
	role *kdv1.Role,
	roleStatus *kdv1.RoleStatus,
) (*appsv1.StatefulSet, error) {
	statefulSet, err := getStatefulset(
		reqLogger,
		cr,
		nativeSystemdSupport,
		role,
		roleStatus,
		0,
	)
	if err != nil {
		return nil, err
	}
	return statefulSet, shared.Create(context.TODO(), statefulSet)
}
func UpdateStatefulSetReplicas(
	reqLogger logr.Logger,
	cr *kdv1.KubeDirectorCluster,
	replicas int32,
	statefulSet *appsv1.StatefulSet,
) error {
	*statefulSet.Spec.Replicas = replicas
	err := shared.Update(context.TODO(), statefulSet)
	if err == nil {
		return nil
	}
	if !errors.IsConflict(err) {
		shared.LogError(
			reqLogger,
			err,
			cr,
			shared.EventReasonNoEvent,
			"failed to update statefulset",
		)
		return err
	}
	name := types.NamespacedName{
		Namespace: statefulSet.Namespace,
		Name:      statefulSet.Name,
	}
	*statefulSet = appsv1.StatefulSet{}
	err = shared.Get(context.TODO(), name, statefulSet)
	if err != nil {
		shared.LogError(
			reqLogger,
			err,
			cr,
			shared.EventReasonNoEvent,
			"failed to retrieve statefulset",
		)
		return err
	}
	*statefulSet.Spec.Replicas = replicas
	err = shared.Update(context.TODO(), statefulSet)
	if err != nil {
		shared.LogError(
			reqLogger,
			err,
			cr,
			shared.EventReasonNoEvent,
			"failed to update statefulset",
		)
	}
	return err
}
func UpdateStatefulSetNonReplicas(
	reqLogger logr.Logger,
	cr *kdv1.KubeDirectorCluster,
	role *kdv1.Role,
	statefulSet *appsv1.StatefulSet,
) error {
	if role == nil {
		return nil
	}
	if shared.OwnerReferencesPresent(cr, statefulSet.OwnerReferences) {
		return nil
	}
	shared.LogInfof(
		reqLogger,
		cr,
		shared.EventReasonNoEvent,
		"repairing owner ref on statefulset{%s}",
		statefulSet.Name,
	)
	patchedRes := *statefulSet
	patchedRes.OwnerReferences = shared.OwnerReferences(cr)
	patchErr := shared.Patch(
		context.TODO(),
		statefulSet,
		&patchedRes,
	)
	return patchErr
}
func DeleteStatefulSet(
	namespace string,
	statefulSetName string,
) error {
	toDelete := &appsv1.StatefulSet{
		TypeMeta: metav1.TypeMeta{
			Kind:       "StatefulSet",
			APIVersion: "apps/v1",
		},
		ObjectMeta: metav1.ObjectMeta{
			Name:      statefulSetName,
			Namespace: namespace,
		},
	}
	return shared.Delete(context.TODO(), toDelete)
}
func getStatefulset(
	reqLogger logr.Logger,
	cr *kdv1.KubeDirectorCluster,
	nativeSystemdSupport bool,
	role *kdv1.Role,
	roleStatus *kdv1.RoleStatus,
	replicas int32,
) (*appsv1.StatefulSet, error) {
	labels := labelsForStatefulSet(cr, role)
	podLabels := labelsForPod(cr, role)
	annotations := annotationsForStatefulSet(cr, role)
	podAnnotations := annotationsForPod(cr, role)
	startupScript := getStartupScript(cr)
	portInfoList, portsErr := catalog.PortsForRole(cr, role.Name)
	if portsErr != nil {
		return nil, portsErr
	}
	var endpointPorts []v1.ContainerPort
	for _, portInfo := range portInfoList {
		containerPort := v1.ContainerPort{
			ContainerPort: portInfo.Port,
			Name:          portInfo.ID,
		}
		endpointPorts = append(endpointPorts, containerPort)
	}
	appPersistDirs, persistErr := catalog.AppPersistDirs(cr, role.Name)
	if persistErr != nil {
		return nil, persistErr
	}
	defaultPersistDirs := &defaultMountFolders
	setupURL, setupURLErr := catalog.AppSetupPackageURL(cr, role.Name)
	if setupURLErr != nil {
		return nil, setupURLErr
	}
	if setupURL != "" {
		defaultPersistDirs = &appConfigDefaultMountFolders
	}
	var maxLen = len(*defaultPersistDirs)
	if appPersistDirs != nil {
		maxLen += len(*appPersistDirs)
	}
	persistDirs := make([]string, len(*defaultPersistDirs), maxLen)
	copy(persistDirs, *defaultPersistDirs)
	if appPersistDirs != nil {
		for _, appDir := range *appPersistDirs {
			isSubDir := false
			for _, defaultDir := range persistDirs {
				rel, _ := filepath.Rel(defaultDir, appDir)
				if !strings.HasPrefix(rel, "..") {
					shared.LogInfof(
						reqLogger,
						cr,
						shared.EventReasonNoEvent,
						"skipping {%s} from volume claim mounts. dir {%s} covers it",
						appDir,
						defaultDir,
					)
					isSubDir = true
					break
				}
			}
			if !isSubDir {
				abs, _ := filepath.Abs(appDir)
				persistDirs = append(persistDirs, abs)
			}
		}
	}
	useServiceAccount := false
	if role.ServiceAccountName != "" {
		useServiceAccount = true
	}
	volumeMounts, volumes, volumesErr := generateVolumeMounts(
		cr,
		role,
		PvcNamePrefix,
		nativeSystemdSupport,
		persistDirs,
	)
	if volumesErr != nil {
		return nil, volumesErr
	}
	var volumeDevices []v1.VolumeDevice
	if role.BlockStorage != nil {
		numDevices := *role.BlockStorage.NumDevices
		for i := int32(0); i < numDevices; i++ {
			deviceID := strconv.FormatInt(int64(i), 10)
			devicePath := *role.BlockStorage.Path + deviceID
			deviceName := blockPvcNamePrefix + deviceID
			volumeDevice := v1.VolumeDevice{
				Name:       deviceName,
				DevicePath: devicePath,
			}
			volumeDevices = append(volumeDevices, volumeDevice)
		}
	}
	imageID, imageErr := catalog.ImageForRole(cr, role.Name)
	if imageErr != nil {
		return nil, imageErr
	}
	securityContext, securityErr := generateSecurityContext(cr)
	if securityErr != nil {
		return nil, securityErr
	}
	vct := getVolumeClaimTemplate(cr, role, PvcNamePrefix)
	sset := &appsv1.StatefulSet{
		TypeMeta: metav1.TypeMeta{
			Kind:       "StatefulSet",
			APIVersion: "apps/v1",
		},
		ObjectMeta: metav1.ObjectMeta{
			Namespace:       cr.Namespace,
			OwnerReferences: shared.OwnerReferences(cr),
			Labels:          labels,
			Annotations:     annotations,
		},
		Spec: appsv1.StatefulSetSpec{
			PodManagementPolicy: appsv1.ParallelPodManagement,
			Replicas:            &replicas,
			ServiceName:         cr.Status.ClusterService,
			Selector: &metav1.LabelSelector{
				MatchLabels: podLabels,
			},
			Template: v1.PodTemplateSpec{
				ObjectMeta: metav1.ObjectMeta{
					Labels:      podLabels,
					Annotations: podAnnotations,
				},
				Spec: v1.PodSpec{
					AutomountServiceAccountToken: &useServiceAccount,
					InitContainers: getInitContainer(
						cr,
						role,
						PvcNamePrefix,
						imageID,
						persistDirs,
					),
					Affinity:           role.Affinity,
					ServiceAccountName: role.ServiceAccountName,
					Containers: []v1.Container{
						{
							Name:            AppContainerName,
							Image:           imageID,
							Resources:       role.Resources,
							Lifecycle:       &v1.Lifecycle{PostStart: &startupScript},
							Ports:           endpointPorts,
							VolumeMounts:    volumeMounts,
							VolumeDevices:   volumeDevices,
							SecurityContext: securityContext,
							Env:             chkModifyEnvVars(role),
							TTY:             hasTTY(cr, role.Name),
							Stdin:           hasSTDIN(cr, role.Name),
						},
					},
					Volumes: volumes,
				},
			},
			VolumeClaimTemplates: vct,
		},
	}
	namingScheme := *cr.Spec.NamingScheme
	if (roleStatus == nil) || (roleStatus.StatefulSet == "") {
		if namingScheme == v1beta1.CrNameRole {
			sset.ObjectMeta.GenerateName = MungObjectName(cr.Name + "-" + role.Name)
			sset.ObjectMeta.GenerateName += "-"
		} else if namingScheme == v1beta1.UID {
			sset.ObjectMeta.GenerateName = statefulSetNamePrefix
		}
	} else {
		sset.ObjectMeta.Name = roleStatus.StatefulSet
	}
	return sset, nil
}
func chkModifyEnvVars(
	role *kdv1.Role,
) (envVar []v1.EnvVar) {
	envVar = role.EnvVars
	rsrcmap := role.Resources.Requests
	if quantity, found := rsrcmap[nvidiaGpuResourceName]; found == true && quantity.IsZero() != true {
		return envVar
	}
	envVarToAdd := v1.EnvVar{
		Name:  nvidiaGpuVisWorkaroundEnvVarName,
		Value: nvidiaGpuVisWorkaroundEnvVarValue,
	}
	envVar = append(envVar, envVarToAdd)
	return
}
func getInitContainer(
	cr *kdv1.KubeDirectorCluster,
	role *kdv1.Role,
	pvcNamePrefix string,
	imageID string,
	persistDirs []string,
) (initContainer []v1.Container) {
	var rootUID int64
	if role.Storage == nil {
		return
	}
	initVolumeMounts := generateInitVolumeMounts(pvcNamePrefix)
	cpus, _ := resource.ParseQuantity("1")
	mem, _ := resource.ParseQuantity("512Mi")
	initContainer = []v1.Container{
		{
			Args: []string{
				"-c",
				generateInitContainerLaunch(persistDirs),
			},
			Command: []string{
				"/bin/bash",
			},
			Image: imageID,
			Name:  initContainerName,
			Resources: v1.ResourceRequirements{
				Limits: v1.ResourceList{
					"cpu":    cpus,
					"memory": mem,
				},
				Requests: v1.ResourceList{
					"cpu":    cpus,
					"memory": mem,
				},
			},
			SecurityContext: &v1.SecurityContext{
				RunAsUser: &rootUID,
			},
			VolumeMounts: initVolumeMounts,
		},
	}
	return
}
func getVolumeClaimTemplate(
	cr *kdv1.KubeDirectorCluster,
	role *kdv1.Role,
	pvcNamePrefix string,
) (volTemplate []v1.PersistentVolumeClaim) {
	if role.Storage != nil {
		volSize, _ := resource.ParseQuantity(role.Storage.Size)
		volClaim := v1.PersistentVolumeClaim{
			ObjectMeta: metav1.ObjectMeta{
				Name: pvcNamePrefix,
				Annotations: map[string]string{
					storageClassName: *role.Storage.StorageClass,
				},
			},
			Spec: v1.PersistentVolumeClaimSpec{
				AccessModes: []v1.PersistentVolumeAccessMode{
					v1.ReadWriteOnce,
				},
				Resources: v1.ResourceRequirements{
					Requests: v1.ResourceList{
						v1.ResourceStorage: volSize,
					},
				},
			},
		}
		volTemplate = append(volTemplate, volClaim)
	}
	if role.BlockStorage != nil {
		block := v1.PersistentVolumeBlock
		blockVolSize, _ := resource.ParseQuantity(defaultBlockDeviceSize)
		if role.BlockStorage.Size != nil {
			blockVolSize, _ = resource.ParseQuantity(*role.BlockStorage.Size)
		}
		numDevices := *role.BlockStorage.NumDevices
		for i := int32(0); i < numDevices; i++ {
			deviceID := strconv.FormatInt(int64(i), 10)
			deviceName := blockPvcNamePrefix + deviceID
			blockClaim := v1.PersistentVolumeClaim{
				ObjectMeta: metav1.ObjectMeta{
					Name: deviceName,
					Annotations: map[string]string{
						storageClassName: *role.BlockStorage.StorageClass,
					},
				},
				Spec: v1.PersistentVolumeClaimSpec{
					AccessModes: []v1.PersistentVolumeAccessMode{
						v1.ReadWriteOnce,
					},
					Resources: v1.ResourceRequirements{
						Requests: v1.ResourceList{
							v1.ResourceStorage: blockVolSize,
						},
					},
					StorageClassName: role.BlockStorage.StorageClass,
					VolumeMode: &block,
				},
			}
			volTemplate = append(volTemplate, blockClaim)
		}
	}
	return volTemplate
}
func getStartupScript(
	cr *kdv1.KubeDirectorCluster,
) v1.Handler {
	return v1.Handler{
		Exec: &v1.ExecAction{
			Command: []string{
				"/bin/bash",
				"-c",
				"exec 2>>/tmp/kd-postcluster.log; set -x;" +
					"Retries=60; while [[ $Retries && ! -s /etc/resolv.conf ]]; do " +
					"sleep 1; Retries=$(expr $Retries - 1); done; " +
					"sed \"s/^search \\([^ ]\\+\\)/search " +
					cr.Status.ClusterService +
					".\\1 \\1/\" /etc/resolv.conf > /tmp/resolv.conf.new && " +
					"cat /tmp/resolv.conf.new > /etc/resolv.conf;" +
					"rm /tmp/resolv.conf.new;" +
					"chmod 755 /run;" +
					"exit 0",
			},
		},
	}
}
func generateInitContainerLaunch(persistDirs []string) string {
	launchCmd := "! [ -f /mnt" + kubedirectorInit + " ]" + " && " +
		"cp --parent -ax " + strings.Join(persistDirs, " ") +
		" /mnt; touch /mnt" + kubedirectorInit
	return launchCmd
}
func generateSecretVolume(
	secret *kdv1.KDSecret,
) ([]v1.VolumeMount, []v1.Volume) {
	if secret != nil {
		secretVolName := "secret-vol-" + secret.Name
		secretVolumeSource := v1.SecretVolumeSource{
			SecretName:  secret.Name,
			DefaultMode: secret.DefaultMode,
		}
		return []v1.VolumeMount{
				v1.VolumeMount{
					Name:      secretVolName,
					MountPath: secret.MountPath,
					ReadOnly:  secret.ReadOnly,
				},
			}, []v1.Volume{
				v1.Volume{
					Name: secretVolName,
					VolumeSource: v1.VolumeSource{
						Secret: &secretVolumeSource,
					},
				},
			}
	}
	return []v1.VolumeMount{}, []v1.Volume{}
}
func generateVolumeMounts(
	cr *kdv1.KubeDirectorCluster,
	role *kdv1.Role,
	pvcNamePrefix string,
	nativeSystemdSupport bool,
	persistDirs []string,
) ([]v1.VolumeMount, []v1.Volume, error) {
	var volumeMounts []v1.VolumeMount
	var volumes []v1.Volume
	if role.Storage != nil {
		volumeMounts = generateClaimMounts(pvcNamePrefix, persistDirs)
	}
	tmpfsVolMnts, tmpfsVols := generateTmpfsSupport(cr)
	volumeMounts = append(volumeMounts, tmpfsVolMnts...)
	volumes = append(volumes, tmpfsVols...)
	secretVolMnts, secretVols := generateSecretVolume(role.Secret)
	volumeMounts = append(volumeMounts, secretVolMnts...)
	volumes = append(volumes, secretVols...)
	isSystemdReqd, err := catalog.SystemdRequired(cr)
	if err != nil {
		return volumeMounts, volumes, err
	}
	if isSystemdReqd && !nativeSystemdSupport {
		cgroupVolMnts, cgroupVols := generateSystemdSupport(cr)
		volumeMounts = append(volumeMounts, cgroupVolMnts...)
		volumes = append(volumes, cgroupVols...)
	}
	return volumeMounts, volumes, nil
}
func generateClaimMounts(
	pvcNamePrefix string,
	persistDirs []string,
) []v1.VolumeMount {
	var volumeMounts []v1.VolumeMount
	for _, folder := range persistDirs {
		volumeMount := v1.VolumeMount{
			MountPath: folder,
			Name:      pvcNamePrefix,
			ReadOnly:  false,
			SubPath:   folder[1:],
		}
		volumeMounts = append(volumeMounts, volumeMount)
	}
	return volumeMounts
}
func generateInitVolumeMounts(
	pvcNamePrefix string,
) []v1.VolumeMount {
	return []v1.VolumeMount{
		v1.VolumeMount{
			MountPath: "/mnt",
			Name:      pvcNamePrefix,
			ReadOnly:  false,
		},
	}
}
func generateSystemdSupport(
	cr *kdv1.KubeDirectorCluster,
) ([]v1.VolumeMount, []v1.Volume) {
	cgroupFsName := "cgroupfs"
	systemdFsName := "systemd"
	volumeMounts := []v1.VolumeMount{
		v1.VolumeMount{
			Name:      cgroupFsName,
			MountPath: cgroupFSVolume,
			ReadOnly:  true,
		},
		v1.VolumeMount{
			Name:      systemdFsName,
			MountPath: systemdFSVolume,
		},
	}
	volumes := []v1.Volume{
		v1.Volume{
			Name: cgroupFsName,
			VolumeSource: v1.VolumeSource{
				HostPath: &v1.HostPathVolumeSource{
					Path: cgroupFSVolume,
				},
			},
		},
		v1.Volume{
			Name: systemdFsName,
			VolumeSource: v1.VolumeSource{
				HostPath: &v1.HostPathVolumeSource{
					Path: systemdFSVolume,
				},
			},
		},
	}
	return volumeMounts, volumes
}