Commit message: Fix security issue in _prolog_error().

Fix security issue caused by insecure file path handling triggered by
the failure of a Prolog script. To exploit this a user needs to
anticipate or cause the Prolog to fail for their job.

(This commit is slightly different from the fix to the 15.08 branch.)

CVE-2016-10030.
CVE: CVE-2016-10030
CWE: CWE-284
Vulnerability description: The _prolog_error function in slurmd/req.c in Slurm before 15.08.13, 16.x before 16.05.7, and 17.x before 17.02.0-pre4 has a vulnerability in how the slurmd daemon informs users of a Prolog failure on a compute node. That vulnerability could allow a user to assume control of an arbitrary file on the system. Any exploitation of this is dependent on the user being able to cause or anticipate the failure (non-zero return code) of a Prolog script that their job would run on. This issue affects all Slurm versions from 0.6.0 (September 2005) to present. Workarounds to prevent exploitation of this are to either disable your Prolog script, or modify it such that it always returns 0 ("success") and adjust it to set the node as down using scontrol instead of relying on the slurmd to handle that automatically. If you do not have a Prolog set you are unaffected by this issue.

Project: slurm
File: req.c

_build_env(job_env_t *job_env)
{
	char **env = xmalloc(sizeof(char *));
	bool user_name_set = 0;

	env[0]  = NULL;
	if (!valid_spank_job_env(job_env->spank_job_env,
				 job_env->spank_job_env_size,
				 job_env->uid)) {
		/* If SPANK job environment is bad, log it and do not use */
		job_env->spank_job_env_size = 0;
		job_env->spank_job_env = (char **) NULL;
	}
	if (job_env->spank_job_env_size) {
		env_array_merge_spank(&env,
				      (const char **) job_env->spank_job_env);
	}

	slurm_mutex_lock(&conf->config_mutex);
	setenvf(&env, "SLURMD_NODENAME", "%s", conf->node_name);
	setenvf(&env, "SLURM_CONF", conf->conffile);
	slurm_mutex_unlock(&conf->config_mutex);

	setenvf(&env, "SLURM_CLUSTER_NAME", "%s", conf->cluster_name);
	setenvf(&env, "SLURM_JOB_ID", "%u", job_env->jobid);
	setenvf(&env, "SLURM_JOB_UID",   "%u", job_env->uid);

#ifndef HAVE_NATIVE_CRAY
	/* uid_to_string on a cray is a heavy call, so try to avoid it */
	if (!job_env->user_name) {
		job_env->user_name = uid_to_string(job_env->uid);
		user_name_set = 1;
	}
#endif

	setenvf(&env, "SLURM_JOB_USER", "%s", job_env->user_name);
	if (user_name_set)
		xfree(job_env->user_name);

	setenvf(&env, "SLURM_JOBID", "%u", job_env->jobid);
	setenvf(&env, "SLURM_UID",   "%u", job_env->uid);
	if (job_env->node_list)
		setenvf(&env, "SLURM_NODELIST", "%s", job_env->node_list);

	if (job_env->partition)
		setenvf(&env, "SLURM_JOB_PARTITION", "%s", job_env->partition);

	if (job_env->resv_id) {
#if defined(HAVE_BG)
		setenvf(&env, "MPIRUN_PARTITION", "%s", job_env->resv_id);
# ifdef HAVE_BGP
		/* Needed for HTC jobs */
		setenvf(&env, "SUBMIT_POOL", "%s", job_env->resv_id);
# endif
#elif defined(HAVE_ALPS_CRAY)
		setenvf(&env, "BASIL_RESERVATION_ID", "%s", job_env->resv_id);
#endif
	}
	return env;
}

Project: slurm
File: req.c

static int _compare_starting_steps(void *listentry, void *key)
{
	starting_step_t *step0 = (starting_step_t *)listentry;
	starting_step_t *step1 = (starting_step_t *)key;

	if (step1->step_id != NO_VAL)
		return (step0->job_id  == step1->job_id &&
			step0->step_id == step1->step_id);
	else
		return (step0->job_id  == step1->job_id);
}

Project: slurm
File: req.c

static void _convert_job_mem(slurm_msg_t *msg)
{
	prolog_launch_msg_t *req = (prolog_launch_msg_t *)msg->data;
	slurm_cred_arg_t arg;
	hostset_t j_hset = NULL;
	int rc, hi, host_index, job_cpus;
	int i, i_first_bit = 0, i_last_bit = 0;

	rc = slurm_cred_verify(conf->vctx, req->cred, &arg,
			       msg->protocol_version);
	if (rc < 0) {
		error("%s: slurm_cred_verify failed: %m", __func__);
		req->nnodes = 1;	/* best guess */
		return;
	}

	req->nnodes = arg.job_nhosts;

	if (arg.job_mem_limit == 0)
		goto fini;
	if ((arg.job_mem_limit & MEM_PER_CPU) == 0) {
		req->job_mem_limit = arg.job_mem_limit;
		goto fini;
	}

	/* Assume 1 CPU on error */
	req->job_mem_limit = arg.job_mem_limit & (~MEM_PER_CPU);

	if (!(j_hset = hostset_create(arg.job_hostlist))) {
		error("%s: Unable to parse credential hostlist: `%s'",
		      __func__, arg.step_hostlist);
		goto fini;
	}
	host_index = hostset_find(j_hset, conf->node_name);
	hostset_destroy(j_hset);

	hi = host_index + 1;	/* change from 0-origin to 1-origin */
	for (i = 0; hi; i++) {
		if (hi > arg.sock_core_rep_count[i]) {
			i_first_bit += arg.sockets_per_node[i] *
				       arg.cores_per_socket[i] *
				       arg.sock_core_rep_count[i];
			i_last_bit = i_first_bit +
				     arg.sockets_per_node[i] *
				     arg.cores_per_socket[i] *
				     arg.sock_core_rep_count[i];
			hi -= arg.sock_core_rep_count[i];
		} else {
			i_first_bit += arg.sockets_per_node[i] *
				       arg.cores_per_socket[i] * (hi - 1);
			i_last_bit = i_first_bit +
				     arg.sockets_per_node[i] *
				     arg.cores_per_socket[i];
			break;
		}
	}

	/* Now count the allocated processors on this node */
	job_cpus = 0;
	for (i = i_first_bit; i < i_last_bit; i++) {
		if (bit_test(arg.job_core_bitmap, i))
			job_cpus++;
	}

	/* NOTE: alloc_lps is the count of allocated resources
	 * (typically cores). Convert to CPU count as needed */
	if (i_last_bit > i_first_bit) {
		i = conf->cpus / (i_last_bit - i_first_bit);
		if (i > 1)
			job_cpus *= i;
	}

	req->job_mem_limit *= job_cpus;

fini:	slurm_cred_free_args(&arg);
}

Project: slurm
File: req.c

_destroy_env(char **env)
{
	int i=0;

	if (env) {
		for(i=0; env[i]; i++) {
			xfree(env[i]);
		}
		xfree(env);
	}

	return;
}

Project: slurm
File: req.c

static void _fb_wrunlock(void)
{
	slurm_mutex_lock(&file_bcast_mutex);
	fb_write_lock--;
	pthread_cond_broadcast(&file_bcast_cond);
	slurm_mutex_unlock(&file_bcast_mutex);
}

Project: slurm

static int _file_bcast_register_file(slurm_msg_t *msg,
				     file_bcast_info_t *key)
{
	file_bcast_msg_t *req = msg->data;
	int fd, flags, rc;
	int pipe[2];
	gids_t *gids;
	pid_t child;
	file_bcast_info_t *file_info;

	if (!(gids = _gids_cache_lookup(req->user_name, key->gid))) {
		error("sbcast: gids_cache_lookup for %s failed", req->user_name);
		return SLURM_ERROR;
	}

	if ((rc = container_g_create(key->job_id))) {
		error("sbcast: container_g_create(%u): %m", key->job_id);
		_dealloc_gids(gids);
		return rc;
	}

	/* child process will setuid to the user, register the process
	 * with the container, and open the file for us. */

	if (socketpair(AF_UNIX, SOCK_DGRAM, 0, pipe) != 0) {
		error("%s: Failed to open pipe: %m", __func__);
		_dealloc_gids(gids);
		return SLURM_ERROR;
	}

	child = fork();
	if (child == -1) {
		error("sbcast: fork failure");
		_dealloc_gids(gids);
		close(pipe[0]);
		close(pipe[1]);
		return errno;
	} else if (child > 0) {
		/* get fd back from pipe */
		close(pipe[0]);
		waitpid(child, &rc, 0);
		_dealloc_gids(gids);
		if (rc) {
			close(pipe[1]);
			return WEXITSTATUS(rc);
		}

		fd = _receive_fd(pipe[1]);
		close(pipe[1]);

		file_info = xmalloc(sizeof(file_bcast_info_t));
		file_info->fd = fd;
		file_info->fname = xstrdup(req->fname);
		file_info->uid = key->uid;
		file_info->gid = key->gid;
		file_info->job_id = key->job_id;
		file_info->start_time = time(NULL);

		_fb_wrlock();
		list_append(file_bcast_list, file_info);
		_fb_wrunlock();

		return SLURM_SUCCESS;
	}

	/* child process below here */

	close(pipe[1]);

	/* container_g_add_pid needs to be called in the
	   forked process part of the fork to avoid a race
	   condition where if this process makes a file or
	   detacts itself from a child before we add the pid
	   to the container in the parent of the fork.
	*/
	if (container_g_add_pid(key->job_id, getpid(), key->uid)) {
		error("container_g_add_pid(%u): %m", key->job_id);
		exit(SLURM_ERROR);
	}

	/* The child actually performs the I/O and exits with
	 * a return code, do not return! */

	/*********************************************************************\
	 * NOTE: It would be best to do an exec() immediately after the fork()
	 * in order to help prevent a possible deadlock in the child process
	 * due to locks being set at the time of the fork and being freed by
	 * the parent process, but not freed by the child process. Performing
	 * the work inline is done for simplicity. Note that the logging
	 * performed by error() should be safe due to the use of
	 * atfork_install_handlers() as defined in src/common/log.c.
	 * Change the code below with caution.
	\*********************************************************************/

	if (setgroups(gids->ngids, gids->gids) < 0) {
		error("sbcast: uid: %u setgroups failed: %m", key->uid);
		exit(errno);
	}
	_dealloc_gids(gids);

	if (setgid(key->gid) < 0) {
		error("sbcast: uid:%u setgid(%u): %m", key->uid, key->gid);
		exit(errno);
	}
	if (setuid(key->uid) < 0) {
		error("sbcast: getuid(%u): %m", key->uid);
		exit(errno);
	}

	flags = O_WRONLY | O_CREAT;
	if (req->force)
		flags |= O_TRUNC;
	else
		flags |= O_EXCL;

	fd = open(key->fname, flags, 0700);
	if (fd == -1) {
		error("sbcast: uid:%u can't open `%s`: %m",
		      key->uid, key->fname);
		exit(errno);
	}
	_send_back_fd(pipe[0], fd);
	close(fd);
	exit(SLURM_SUCCESS);
}

Project: slurm
File: req.c

static int _find_waiter(struct waiter *w, uint32_t *jp)
{
	return (w->jobid == *jp);
}

Project: slurm
File: req.c

static void _handle_old_batch_job_launch(slurm_msg_t *msg)
{
	if (msg->msg_type != REQUEST_BATCH_JOB_LAUNCH) {
		error("_handle_batch_job_launch: "
		      "Invalid response msg_type (%u)", msg->msg_type);
		return;
	}

	/* (resp_msg.msg_type == REQUEST_BATCH_JOB_LAUNCH) */
	debug2("Processing RPC: REQUEST_BATCH_JOB_LAUNCH");
	last_slurmctld_msg = time(NULL);
	_rpc_batch_job(msg, false);
	slurm_free_job_launch_msg(msg->data);
	msg->data = NULL;

}

Project: slurm
File: req.c

_rpc_complete_batch(slurm_msg_t *msg)
{
	int		i, rc, msg_rc;
	slurm_msg_t	resp_msg;
	uid_t           uid    = g_slurm_auth_get_uid(msg->auth_cred,
						      conf->auth_info);
	complete_batch_script_msg_t *req = msg->data;
	static int	running_serial = -1;
	uint16_t msg_type;

	if (running_serial == -1) {
		char *select_type = slurm_get_select_type();
		if (!xstrcmp(select_type, "select/serial"))
			running_serial = 1;
		else
			running_serial = 0;
		xfree(select_type);
	}

	if (!_slurm_authorized_user(uid)) {
		error("Security violation: complete_batch(%u) from uid %d",
		      req->job_id, uid);
		if (msg->conn_fd >= 0)
			slurm_send_rc_msg(msg, ESLURM_USER_ID_MISSING);
		return;
	}

	slurm_send_rc_msg(msg, SLURM_SUCCESS);

	if (running_serial) {
		_rpc_terminate_batch_job(
			req->job_id, req->user_id, req->node_name);
		msg_type = REQUEST_COMPLETE_BATCH_JOB;
	} else
		msg_type = msg->msg_type;

	for (i = 0; i <= MAX_RETRY; i++) {
		if (conf->msg_aggr_window_msgs > 1) {
			slurm_msg_t *req_msg =
				xmalloc_nz(sizeof(slurm_msg_t));
			slurm_msg_t_init(req_msg);
			req_msg->msg_type = msg_type;
			req_msg->data = msg->data;
			msg->data = NULL;

			msg_aggr_add_msg(req_msg, 1,
					 _handle_old_batch_job_launch);
			return;
		} else {
			slurm_msg_t req_msg;
			slurm_msg_t_init(&req_msg);
			req_msg.msg_type = msg_type;
			req_msg.data	 = msg->data;
			msg_rc = slurm_send_recv_controller_msg(
				&req_msg, &resp_msg);

			if (msg_rc == SLURM_SUCCESS)
				break;
		}
		info("Retrying job complete RPC for job %u", req->job_id);
		sleep(RETRY_DELAY);
	}
	if (i > MAX_RETRY) {
		error("Unable to send job complete message: %m");
		return;
	}

	if (resp_msg.msg_type == RESPONSE_SLURM_RC) {
		last_slurmctld_msg = time(NULL);
		rc = ((return_code_msg_t *) resp_msg.data)->return_code;
		slurm_free_return_code_msg(resp_msg.data);
		if (rc) {
			error("complete_batch for job %u: %s", req->job_id,
			      slurm_strerror(rc));
		}
		return;
	}

	_handle_old_batch_job_launch(&resp_msg);
}

Project: slurm
File: req.c

_rpc_list_pids(slurm_msg_t *msg)
{
	job_step_id_msg_t *req = (job_step_id_msg_t *)msg->data;
	slurm_msg_t        resp_msg;
	job_step_pids_t *resp = NULL;
	int fd;
	uid_t req_uid;
	uid_t job_uid;
	uint16_t protocol_version = 0;

	debug3("Entering _rpc_list_pids");
	/* step completion messages are only allowed from other slurmstepd,
	 * so only root or SlurmUser is allowed here */
	req_uid = g_slurm_auth_get_uid(msg->auth_cred, conf->auth_info);

	job_uid = _get_job_uid(req->job_id);

	if ((int)job_uid < 0) {
		error("stat_pid for invalid job_id: %u",
		      req->job_id);
		if (msg->conn_fd >= 0)
			slurm_send_rc_msg(msg, ESLURM_INVALID_JOB_ID);
		return  ESLURM_INVALID_JOB_ID;
	}

	/*
	 * check that requesting user ID is the SLURM UID or root
	 */
	if ((req_uid != job_uid)
	    && (!_slurm_authorized_user(req_uid))) {
		error("stat_pid from uid %ld for job %u "
		      "owned by uid %ld",
		      (long) req_uid, req->job_id, (long) job_uid);

		if (msg->conn_fd >= 0) {
			slurm_send_rc_msg(msg, ESLURM_USER_ID_MISSING);
			return ESLURM_USER_ID_MISSING;/* or bad in this case */
		}
	}

	resp = xmalloc(sizeof(job_step_pids_t));
	slurm_msg_t_copy(&resp_msg, msg);
	resp->node_name = xstrdup(conf->node_name);
	resp->pid_cnt = 0;
	resp->pid = NULL;
	fd = stepd_connect(conf->spooldir, conf->node_name,
			   req->job_id, req->step_id, &protocol_version);
	if (fd == -1) {
		error("stepd_connect to %u.%u failed: %m",
		      req->job_id, req->step_id);
		slurm_send_rc_msg(msg, ESLURM_INVALID_JOB_ID);
		slurm_free_job_step_pids(resp);
		return  ESLURM_INVALID_JOB_ID;

	}

	if (stepd_list_pids(fd, protocol_version,
			    &resp->pid, &resp->pid_cnt) == SLURM_ERROR) {
		debug("No pids for nonexistent job %u.%u requested",
		      req->job_id, req->step_id);
	}

	close(fd);

	resp_msg.msg_type = RESPONSE_JOB_STEP_PIDS;
	resp_msg.data     = resp;

	slurm_send_node_msg(msg->conn_fd, &resp_msg);
	slurm_free_job_step_pids(resp);
	return SLURM_SUCCESS;
}

Project: slurm
File: req.c

_rpc_signal_job(slurm_msg_t *msg)
{
	signal_job_msg_t *req = msg->data;
	uid_t req_uid = g_slurm_auth_get_uid(msg->auth_cred,
					     conf->auth_info);
	uid_t job_uid;
	List steps;
	ListIterator i;
	step_loc_t *stepd = NULL;
	int step_cnt  = 0;
	int fd;

	debug("_rpc_signal_job, uid = %d, signal = %d", req_uid, req->signal);
	job_uid = _get_job_uid(req->job_id);
	if ((int)job_uid < 0)
		goto no_job;

	/*
	 * check that requesting user ID is the SLURM UID or root
	 */
	if ((req_uid != job_uid) && (!_slurm_authorized_user(req_uid))) {
		error("Security violation: kill_job(%u) from uid %d",
		      req->job_id, req_uid);
		if (msg->conn_fd >= 0) {
			slurm_send_rc_msg(msg, ESLURM_USER_ID_MISSING);
			if (slurm_close(msg->conn_fd) < 0)
				error ("_rpc_signal_job: close(%d): %m",
				       msg->conn_fd);
			msg->conn_fd = -1;
		}
		return;
	}

	/*
	 * Loop through all job steps for this job and signal the
	 * step's process group through the slurmstepd.
	 */
	steps = stepd_available(conf->spooldir, conf->node_name);
	i = list_iterator_create(steps);
	while ((stepd = list_next(i))) {
		if (stepd->jobid != req->job_id) {
			/* multiple jobs expected on shared nodes */
			debug3("Step from other job: jobid=%u (this jobid=%u)",
			       stepd->jobid, req->job_id);
			continue;
		}

		if (stepd->stepid == SLURM_BATCH_SCRIPT) {
			debug2("batch script itself not signalled");
			continue;
		}

		step_cnt++;

		fd = stepd_connect(stepd->directory, stepd->nodename,
				   stepd->jobid, stepd->stepid,
				   &stepd->protocol_version);
		if (fd == -1) {
			debug3("Unable to connect to step %u.%u",
			       stepd->jobid, stepd->stepid);
			continue;
		}

		debug2("  signal %d to job %u.%u",
		       req->signal, stepd->jobid, stepd->stepid);
		if (stepd_signal_container(
			    fd, stepd->protocol_version, req->signal) < 0)
			debug("signal jobid=%u failed: %m", stepd->jobid);
		close(fd);
	}
	list_iterator_destroy(i);
	FREE_NULL_LIST(steps);

no_job:
	if (step_cnt == 0) {
		debug2("No steps in jobid %u to send signal %d",
		       req->job_id, req->signal);
	}

	/*
	 *  At this point, if connection still open, we send controller
	 *   a "success" reply to indicate that we've recvd the msg.
	 */
	if (msg->conn_fd >= 0) {
		slurm_send_rc_msg(msg, SLURM_SUCCESS);
		if (slurm_close(msg->conn_fd) < 0)
			error ("_rpc_signal_job: close(%d): %m", msg->conn_fd);
		msg->conn_fd = -1;
	}
}

Project: slurm
File: req.c

_rpc_suspend_job(slurm_msg_t *msg)
{
	int time_slice = -1;
	suspend_int_msg_t *req = msg->data;
	uid_t req_uid = g_slurm_auth_get_uid(msg->auth_cred,
					     conf->auth_info);
	List steps;
	ListIterator i;
	step_loc_t *stepd;
	int step_cnt  = 0;
	int rc = SLURM_SUCCESS;
	DEF_TIMERS;

	if (time_slice == -1)
		time_slice = slurm_get_time_slice();
	if ((req->op != SUSPEND_JOB) && (req->op != RESUME_JOB)) {
		error("REQUEST_SUSPEND_INT: bad op code %u", req->op);
		rc = ESLURM_NOT_SUPPORTED;
	}

	/*
	 * check that requesting user ID is the SLURM UID or root
	 */
	if (!_slurm_authorized_user(req_uid)) {
		error("Security violation: suspend_job(%u) from uid %d",
		      req->job_id, req_uid);
		rc =  ESLURM_USER_ID_MISSING;
	}

	/* send a response now, which will include any errors
	 * detected with the request */
	if (msg->conn_fd >= 0) {
		slurm_send_rc_msg(msg, rc);
		if (slurm_close(msg->conn_fd) < 0)
			error("_rpc_suspend_job: close(%d): %m",
			      msg->conn_fd);
		msg->conn_fd = -1;
	}
	if (rc != SLURM_SUCCESS)
		return;

	/* now we can focus on performing the requested action,
	 * which could take a few seconds to complete */
	debug("_rpc_suspend_job jobid=%u uid=%d action=%s", req->job_id,
	      req_uid, req->op == SUSPEND_JOB ? "suspend" : "resume");

	/* Try to get a thread lock for this job. If the lock
	 * is not available then sleep and try again */
	while (!_get_suspend_job_lock(req->job_id)) {
		debug3("suspend lock sleep for %u", req->job_id);
		usleep(10000);
	}
	START_TIMER;

	/* Defer suspend until job prolog and launch complete */
	if (req->op == SUSPEND_JOB)
		_launch_complete_wait(req->job_id);

	if ((req->op == SUSPEND_JOB) && (req->indf_susp))
		switch_g_job_suspend(req->switch_info, 5);

	/* Release or reclaim resources bound to these tasks (task affinity) */
	if (req->op == SUSPEND_JOB) {
		(void) task_g_slurmd_suspend_job(req->job_id);
	} else {
		(void) task_g_slurmd_resume_job(req->job_id);
	}

	/*
	 * Loop through all job steps and call stepd_suspend or stepd_resume
	 * as appropriate. Since the "suspend" action may contains a sleep
	 * (if the launch is in progress) suspend multiple jobsteps in parallel.
	 */
	steps = stepd_available(conf->spooldir, conf->node_name);
	i = list_iterator_create(steps);

	while (1) {
		int x, fdi, fd[NUM_PARALLEL_SUSP_STEPS];
		uint16_t protocol_version[NUM_PARALLEL_SUSP_STEPS];

		fdi = 0;
		while ((stepd = list_next(i))) {
			if (stepd->jobid != req->job_id) {
				/* multiple jobs expected on shared nodes */
				debug3("Step from other job: jobid=%u "
				       "(this jobid=%u)",
				       stepd->jobid, req->job_id);
				continue;
			}
			step_cnt++;

			fd[fdi] = stepd_connect(stepd->directory,
						stepd->nodename, stepd->jobid,
						stepd->stepid,
						&protocol_version[fdi]);
			if (fd[fdi] == -1) {
				debug3("Unable to connect to step %u.%u",
				       stepd->jobid, stepd->stepid);
				continue;
			}

			fdi++;
			if (fdi >= NUM_PARALLEL_SUSP_STEPS)
				break;
		}
		/* check for open connections */
		if (fdi == 0)
			break;

		if (req->op == SUSPEND_JOB) {
			int susp_fail_count = 0;
			/* The suspend RPCs are processed in parallel for
			 * every step in the job */
			for (x = 0; x < fdi; x++) {
				(void) stepd_suspend(fd[x],
						     protocol_version[x],
						     req, 0);
			}
			for (x = 0; x < fdi; x++) {
				if (stepd_suspend(fd[x],
						  protocol_version[x],
						  req, 1) < 0) {
					susp_fail_count++;
				} else {
					close(fd[x]);
					fd[x] = -1;
				}
			}
			/* Suspend RPCs can fail at step startup, so retry */
			if (susp_fail_count) {
				sleep(1);
				for (x = 0; x < fdi; x++) {
					if (fd[x] == -1)
						continue;
					(void) stepd_suspend(
						fd[x],
						protocol_version[x],
						req, 0);
					if (stepd_suspend(
						    fd[x],
						    protocol_version[x],
						    req, 1) >= 0)
						continue;
					debug("Suspend of job %u failed: %m",
					      req->job_id);
				}
			}
		} else {
			/* The resume RPCs are processed in parallel for
			 * every step in the job */
			for (x = 0; x < fdi; x++) {
				(void) stepd_resume(fd[x],
						    protocol_version[x],
						    req, 0);
			}
			for (x = 0; x < fdi; x++) {
				if (stepd_resume(fd[x],
						 protocol_version[x],
						 req, 1) < 0) {
					debug("Resume of job %u failed: %m",
					      req->job_id);
				}
			}
		}
		for (x = 0; x < fdi; x++) {
			/* fd may have been closed by stepd_suspend */
			if (fd[x] != -1)
				close(fd[x]);
		}

		/* check for no more jobs */
		if (fdi < NUM_PARALLEL_SUSP_STEPS)
			break;
	}
	list_iterator_destroy(i);
	FREE_NULL_LIST(steps);

	if ((req->op == RESUME_JOB) && (req->indf_susp))
		switch_g_job_resume(req->switch_info, 5);

	_unlock_suspend_job(req->job_id);

	END_TIMER;
	if (DELTA_TIMER >= (time_slice * 1000000)) {
		if (req->op == SUSPEND_JOB) {
			info("Suspend time for job_id %u was %s. "
			     "Configure SchedulerTimeSlice higher.",
			     req->job_id, TIME_STR);
		} else {
			info("Resume time for job_id %u was %s. "
			     "Configure SchedulerTimeSlice higher.",
			     req->job_id, TIME_STR);
		}
	}

	if (step_cnt == 0) {
		debug2("No steps in jobid %u to suspend/resume", req->job_id);
	}
}

Project: slurm
File: req.c

static void _setup_step_complete_msg(slurm_msg_t *msg, void *data)
{
	slurm_msg_t_init(msg);
	msg->msg_type = REQUEST_STEP_COMPLETE;
	msg->data = data;
}