#!/bin/bash
#
# This file is provided under a dual BSD/GPLv2 license.  When using or
# redistributing this file, you may do so under either license.
#
# GPL LICENSE SUMMARY
#
# Copyright(c) 2016 Intel Corporation.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of version 2 of the GNU General Public License as
# published by the Free Software Foundation.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# Contact Information:
# Cong Xu, cong.xu@intel.com
#
# BSD LICENSE
#
# Copyright(c) 2016 Intel Corporation.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in
# the documentation and/or other materials provided with the
# distribution.
# * Neither the name of Intel Corporation nor the names of its
# contributors may be used to endorse or promote products derived
# from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

function usage() {
	cat << EOF
Usage: $0 [-i] [-x] [-y] [-t]
	-i  path to LIOProf rpc tracing logs
	-x  lowest Lustre Client [IB IP Address]
	-y  highest Lustre Client [IB IP Address]
	-t  Type of Operation Code (OPC) (OST_READ 3, OST_WRITE 4)
EOF
	exit 0
}

while getopts ":i:x:y:t:" o; do
	case "${o}" in
		i)
			i=${OPTARG};;
		x)
			x=${OPTARG};;
		y)
			y=${OPTARG};;
		t)
			t=${OPTARG};;
		*)
			usage;;
	esac
done
shift $((OPTIND-1))

if [ -z "${i}" ] || [ -z "${x}" ] || [ -z "${y}" ] || [ -z "${t}" ]; then
	usage
fi

# Cluster Name
cluster_name=$(cut -d- -f1 <<<"${x}")

# Lustre Clients
CLIENT_PRE=$(cut -d. -f 1-3 <<<"${x}")
CLIENT_MIN=$(cut -d. -f4 <<<"${x}")
CLIENT_MAX=$(cut -d. -f4 <<<"${y}")

# Type of Operation Code (OPC) (Defined in lustre/include/lustre/lustre_idl.h)
OPC_TYPE=${t}

# Input directory
IN_PUT=${i}

# Output directory
OUT_PUT=${i}-out
rm -rf $OUT_PUT
mkdir -p $OUT_PUT

for f in ${IN_PUT}/*
do
	echo "Processing ${f}"
	for ((c = $CLIENT_MIN; c <= $CLIENT_MAX; c = c + 1))
	do
		ip=${CLIENT_PRE}.$c
		CUR_OST=$(echo "${f}" | rev | cut -d'/' -f1 | rev)

		cat ${f} | grep "Handling RPC pname" | grep "ll_ost_io" | \
		grep o2ib:${OPC_TYPE} | grep ${ip} | \
		awk 'BEGIN{FS=":"}{print $4}' | sort -n | \
		awk 'BEGIN {count = 0; line = 0; FS="."} {
			if (NR == 1) {curval = $1};
			if($1 <= curval) {
				count = count + 1;
			} else {
				print line "\t" count;
				curval = curval + 1;
				line = line + 1;
				count = 1;
				while(curval < $1) {
					print line "\t" 0;
					curval = curval + 1;
					line = line + 1;
				}
			}
		} END {
			print line "\t" count;
		}' \
		> ${OUT_PUT}/$CUR_OST-Client-$c &
	done
done

# Wait for completion
wait

# If the lines of the files are different, fill the end of files with zeros to
# guarantee all the files have the same number of lines.

# Get max line
MAX_LINE=-1
for f in ${OUT_PUT}/*
do
	LINE=$(wc -l < ${f})
	if [ "$MAX_LINE" -lt "$LINE" ]
	then
		MAX_LINE=${LINE}
	fi
done

# Append zeros
for f in  ${OUT_PUT}/*
do
	LINE=$(wc -l < ${f})
	for ((i = $LINE; i < $MAX_LINE; i = i + 1))
	do
		printf  "0\t0\n" >> ${f} &
	done
done


#!/bin/bash
#
# License:      GNU General Public License (GPL)
#
# lustre_server
#      Description: Manages a lustre_server on a shared storage medium.
#  Original Author: Eric Z. Ayers (eric.ayers@compgen.com)
# Original Release: 25 Oct 2000
#
# Rewritten Filesystem agent to lustre_server agent by bschubert@ddn.com
#
# usage: ./lustre_server {start|stop|status|monitor|validate-all|meta-data}
#
#         OCF parameters are as below:
#               OCF_RESKEY_device
#               OCF_RESKEY_directory
#               OCF_RESKEY_options
#
#OCF_RESKEY_device    : name of block device for the filesystem. e.g. /dev/sda1, /dev/md0
#                       Or a -U or -L option for mount, or an NFS mount specification
#OCF_RESKEY_directory : the mount point for the filesystem
#OCF_RESKEY_options   : options to be given to the mount command via -o
#
#
# NOTE: There is no locking (such as a SCSI reservation) being done here.
#       I would if the SCSI driver could properly maintain the reservation,
#       which it cannot, even with the 'scsi reservation' patch submitted
#       earlier this year by James Bottomley.  The patch minimizes the
#       bus resets caused by a RESERVATION_CONFLICT return, and helps the
#       reservation stay when 2 nodes contend for a reservation,
#       but it does not attempt to recover the reservation in the
#       case of a bus reset.
#
#       What all this means is that if 2 nodes mount the same file system
#       read-write, the filesystem is going to become corrupted. However
#       Lustre provides the Multi-Mount-Protection feature (MMP).
#
#       As a result, you should use this together with the stonith option
#       and redundant, independent communications paths.
#
#       If you don't do this, don't blame us when you scramble your disk.
#
#       Note:  the ServeRAID controller does prohibit concurrent acess
#       In this case, you don't actually need STONITH, but redundant comm is
#       still an excellent idea.
#

#######################################################################
# Initialization:

. ${OCF_ROOT}/resource.d/heartbeat/.ocf-shellfuncs

#######################################################################
HOSTOS=`uname`
FSTYPE="lustre"

# FIXME: OCF_DEBUG officially supported in the the mean time?
#OCF_DEBUG=yes

usage() {
        cat <<-EOT
        usage: $0 {start|stop|status|monitor|validate-all|meta-data}
        EOT
}

meta_data() {
        cat <<END
<?xml version="1.0"?>
<!DOCTYPE resource-agent SYSTEM "ra-api-1.dtd">
<resource-agent name="lustre_server">
<version>1.0</version>

<longdesc lang="en">
  Resource script for lustre_server. It manages a lustre_server on a shared storage medium.
</longdesc>
<shortdesc lang="en">lustre_server resource agent</shortdesc>

<parameters>
  <parameter name="device" required="1">
    <longdesc lang="en">
       The name of block device for the filesystem
  </longdesc>
  <shortdesc lang="en">block device</shortdesc>
  <content type="string" default="" />
</parameter>

<parameter name="directory" required="1">
 <longdesc lang="en">
   The mount point for the filesystem.
 </longdesc>
 <shortdesc lang="en">mount point</shortdesc>
 <content type="string" default="" />
</parameter>

<parameter name="options">
 <longdesc lang="en">
  Any extra options to be given as -o options to mount.
 </longdesc>
 <shortdesc lang="en">options</shortdesc>
 <content type="string" default="" />
</parameter>
</parameters>

<actions>
 <action name="start" timeout="300" />
 <action name="stop" timeout="300" />
 <action name="monitor" depth="0" timeout="300" interval="120" start-delay="10" />
 <action name="validate-all" timeout="5" />
 <action name="meta-data" timeout="5" />
 </actions>
</resource-agent>
END
}

#
#       Make sure the kernel does the right thing with the FS buffers
#       This function should be called after unmounting and before mounting
#       It may not be necessary in 2.4 and later kernels, but it shouldn't hurt
#       anything either...
#
#       It's really a bug that you have to do this at all...
#
flushbufs() {
    if have_binary $BLOCKDEV ; then
       $BLOCKDEV --flushbufs $1
    fi
    return 0
}

# Figure out the real device number of external journals
# we need to provide as mount options, since the block
# device only know major/minor of the the journal device and it
# might/will change dynamically on reboots and between servers
# It would be MUCH better if 'mount' would do it on its own
# and in fact mount is already linked against libblkid to do
# this job
get_external_journal_device()
{
        UUID=`dumpe2fs -h $DEVICE 2>/dev/null | awk '/^Journal UUID/{print $3}'`
        if [ -z "$UUID" ]; then
                # device has internal journal, not need to proceed
                return
        fi

        # prefer /dev/mapper/
        JDEV="`blkid -t UUID=$UUID /dev/mapper/* | awk -F: '{print $1}'`"
        if [ -z "$JDEV" ]; then
                JDEV="`blkid -t UUID=$UUID | awk -F: '{print $1}'`"
        fi

        if [ -z "$JDEV" ]; then
                # know the journal is on an external device, but we can't find it
                ocf_log err "Cannot find device with journal UUID $UUID"
                return $OCF_ERR_GENERIC
        fi

        DEVNUM="`stat -c %02t%02T $JDEV`"
        if [ -z "$DEVNUM" ]; then
                ocf_log err "Failed to retrieve device number of Journal device"
                return $OCF_ERR_GENERIC
        fi

        # add 0x only here, because we couldn't check for an empty string otherwise
        DEVNUM="0x$DEVNUM"

        echo $DEVNUM
        return 0
}

# Take advantage of /proc/mounts if present, use portable mount command
# otherwise. Normalize format to "dev mountpoint fstype".
list_mounts() {
        mtab=/proc/mounts
        if [ ! -f $mtab ]; then
                ocf_log err "$mtab is missing!"
                exit $OCF_ERR_GENERIC
        fi
        cat $mtab | cut -d' ' -f1,2,3
}

lustre_health_check()
{
	check=$(lctl get_param -n health_check 2>&1)
        # on first check the lustre modules are not loaded yet
        if [ $? != 0 ]; then
                return 0
        fi

        if [ "$check" = "healthy" ]; then
                return 0
        else
                ocf_log err "health_check is $check"
                return 1
        fi
}

#
# START: Start up the filesystem
#
lustre_server_start()
{
        lustre_health_check
        if [ $? -ne 0 ]; then
                return ${OCF_ERR_GENERIC}
        fi

        # See if the device is currently mounted
        lustre_server_status >/dev/null 2>&1
        rc=$?
        if [ $rc -ne $OCF_NOT_RUNNING -a $rc -ne $OCF_SUCCESS ]; then
                ocf_log err "Unexpected status ($c) before mounting $DEVICE to $MOUNTPOINT"
        elif [ $rc -eq $OCF_SUCCESS ]; then
                return $OCF_SUCCESS
        fi

        if [ ! -d "$MOUNTPOINT" ] ; then
                ocf_log err "Couldn't find directory  [$MOUNTPOINT] to use as a mount point"
                exit $OCF_ERR_ARGS
        fi

        flushbufs $DEVICE

        jdev=`get_external_journal_device`
        rc=$?
        if [ $rc -ne 0 ]; then
                ocf_log err "Failure to get journal device"
                return $rc
        fi

        if [ -n "$jdev" ]; then
                if [ -n "$options" ]; then
                        options="$options,journal_dev=$jdev"
                else
                        options="-ojournal_dev=$jdev"
                fi
        fi

        # Mount the filesystem.
        cmd="$MOUNT -t $FSTYPE $options $DEVICE $MOUNTPOINT"
        ocf_log info "Running $cmd"
        res=`eval $cmd 2>&1`
        if [ $? -ne 0 ]; then
                ocf_log err "cmd \"$cmd\" failed: $res"
                return $OCF_ERR_GENERIC
        fi

        return 0
}
# end of lustre_server_start

#
# STOP: Unmount the filesystem
#
lustre_server_stop()
{
        # See if the device is currently mounted
        lustre_server_mounted >/dev/null 2>&1
        if [ $? -eq $OCF_NOT_RUNNING ]; then
                # Already unmounted, wonderful.
                rc=$OCF_SUCCESS
        else
                # Umount $MOUNTPOINT.
                ocf_log info "Trying to unmount $MOUNTPOINT"
                cmd="$UMOUNT $MOUNTPOINT"
                ocf_log info "Running $cmd"
                res=`eval $cmd 2>&1`
                rc=$?
                if [ $rc -eq 0 ] ; then
                        rc=$OCF_SUCCESS
                        ocf_log info "unmounted $MOUNTPOINT successfully"
                else
                        rc=$OCF_ERR_GENERIC
                        ocf_log err "\"$cmd\" failed ($rc): $res"
                fi
        fi

        flushbufs $DEVICE

        return $rc
}
# end of lustre_server_stop

#
# MOUNTED: is the filesystem mounted or not?
#
lustre_server_mounted()
{
        if list_mounts | grep -q " $MOUNTPOINT " >/dev/null 2>&1; then
                rc=$OCF_SUCCESS
                msg="$MOUNTPOINT is mounted (running)"
        else
                rc=$OCF_NOT_RUNNING
                msg="$MOUNTPOINT is unmounted (stopped)"
        fi

        # check in all mntdevs if really not mounted
        # lustre bug 21359 (https://bugzilla.lustre.org/show_bug.cgi?id=21359)
        if [ $rc -eq $OCF_NOT_RUNNING ]; then
		dev=$(lctl get_param -n mds.*.mntdev 2>&1)
		if [ $? = 0 ]; then
			MNTDEVS=$dev
		fi
		dev=$(lctl get_param -n obdfilter.*.mntdev 2>&1)
		if [ $? = 0 ]; then
			MNTDEVS="$MNTDEVS $dev"
		fi
		dev=$(lctl get_param -n mgs.MGS.mntdev 2>&1)
		if [ $? = 0 ]; then
			MNTDEVS="$MNTDEVS $dev"
		fi
		for i in $MNTDEVS; do
			if [ "$i" = "$DEVICE" ]; then
                                ocf_log err "Bug21359, /proc/mounts claims device is not mounted, but $i proves this is wrong"
                                rc=$OCF_ERR_GENERIC
                        fi
                done

        fi

        if [ "$OCF_DEBUG" = "yes" ]; then
                ocf_log info "$msg"
        fi

        case "$OP" in
        status) ocf_log info "$msg"
                ;;
        monitor)
                if [ $rc -ne $OCF_SUCCESS ]; then
                        ocf_log err "$msg"
                fi
        esac

        return $rc
}
# end of lustre_server_mounted

#
# STATUS: is the filesystem mounted and healthy or not?
#
lustre_server_status()
{
        lustre_health_check
        if [ $? -ne 0 ]; then
                return ${OCF_ERR_GENERIC}
        fi

        lustre_server_mounted
        rc=$?

        return $rc
}
# end of lustre_server_status

#
#       Check if Lustre is available at all
#
lustre_server_validate_all()
{
	var=$(lctl get_param -n version 2>&1)
	if [ $? != 0 ]; then
		modprobe lustre

		for i in `seq 1 10`; do
			var=$(lctl get_param -n version 2>&1)
			if [ $? != 0 ]; then
				sleep 1
			else
				break
			fi
		done

		var=$(lctl get_param -n version 2>&1)
		if [ $? != 0 ]; then
			ocf_log err "Failed to load the lustre module"
			return $OCF_ERR_GENERIC
		fi
	fi

        return $OCF_SUCCESS
}

# Check the arguments passed to this script
if [ $# -ne 1 ]; then
        usage
        exit $OCF_ERR_ARGS
fi

# Check the OCF_RESKEY_ environment variables...
DEVICE=$OCF_RESKEY_device
if [ ! -z "$OCF_RESKEY_options" ]; then
        options="-o $OCF_RESKEY_options"
fi

OP=$1

if [ "$OCF_DEBUG" = "yes" ]; then
        ocf_log info "OP = $OP"
fi

# These operations do not require instance parameters
case $OP in
meta-data)      meta_data
                exit $OCF_SUCCESS
                ;;
usage)          usage
                exit $OCF_SUCCESS
                ;;
esac

# Normalize instance parameters:

# It is possible that OCF_RESKEY_directory has one or even multiple trailing "/".
# But the output of `mount` and /proc/mounts do not.
if [ -z "$OCF_RESKEY_directory" ]; then
        ocf_log err "Please specify the directory"
        exit $OCF_ERR_ARGS
else
        MOUNTPOINT=$(echo $OCF_RESKEY_directory | sed 's/\/*$//')
        : ${MOUNTPOINT:=/}
        # At this stage, $MOUNTPOINT does not contain trailing "/" unless it is "/"
        # TODO: / mounted via lustre_server sounds dangerous. On stop, we'll
        # kill the whole system. Is that a good idea?
fi

MOUNTPOINT=`echo $MOUNTPOINT | sed -e 's/\s*//'`
if [ -z "$MOUNTPOINT" ]; then
        ocf_log err "Empty mount point!"
        ocf_log err "Please specify the directory"
        exit $OCF_ERR_ARGS
fi

# Check to make sure the utilites are found
check_binary $MODPROBE
check_binary $FSCK
check_binary $MOUNT
check_binary $UMOUNT

if [ "$OP" != "monitor" ]; then
        ocf_log info "Running $OP for $DEVICE on $MOUNTPOINT"
fi

# These operations do not require the clone checking + OCFS2
# initialization.
case $OP in
status|monitor) lustre_server_status
                exit $?
                ;;
validate-all)   lustre_server_validate_all
                exit $?
                ;;
stop)           lustre_server_stop
                exit $?
                ;;
esac

if [ -n "$OCF_RESKEY_CRM_meta_clone" ]; then
        ocf_log err "DANGER! $FSTYPE on $DEVICE is NOT cluster-aware!"
        ocf_log err "DO NOT RUN IT AS A CLONE!"
        ocf_log err "Politely refusing to proceed to avoid data corruption."
        exit $OCF_ERR_GENERIC
fi

case $OP in
start)  lustre_server_start
        ;;
*)      usage
        exit $OCF_ERR_UNIMPLEMENTED
        ;;
esac
exit $?


#!/bin/bash

die() {
	echo -e $* >&2
	echo aborting.. >&2
	exit 1
}

canon() {
	cd $1
	CANON=$PWD
	cd -
}

canon $(dirname $0)
MYDIR=$CANON

while [ ${#*} -gt 1 ]; do
        case "$1" in
                -t)
                        shift;
                        TREE=$1
                        ;;
                -s)
                        shift;
                        SERIES=$1
                        ;;
                *)
			die "unknown argument $1"
                        break;
                        ;;
        esac
        shift;
done

[ -z "$TREE" -o -z "$SERIES" ] && die "I need a tree and series:\n\t$0 -t kernel_dir -s series_name"
[ ! -d $TREE ] && die "kernel tree '$TREE' isn't a directory"
SERIES=$(basename $SERIES)
[ ! -f $MYDIR/series/$SERIES ] && die "no series file '$SERIES'"

canon $TREE
TREE=$CANON

# patch scripts wants a relative path from the linux tree to
# its patch pile :(

MY=$(echo $MYDIR | sed -e 's_^/__')
TR=$(echo $TREE | sed -e 's_^/__')

while true ; do
	M=$(echo $MY | cut -d/ -f 1)
	T=$(echo $TR | cut -d/ -f 1)

	if [ $M != $T ]; then
		break;
	fi

	MY=$(echo $MY | cut -d/ -f 2-)
	TR=$(echo $TR | cut -d/ -f 2-)
done

[ $MY == $MYDIR ] && die "bad! $MY == $MYDIR"

REVERSE=$(revpath $TR)${MY}
ABSINO=$(stat $MYDIR | awk '($3 == "Inode:") {print $4}')
REVINO=`(cd $TREE ; stat $REVERSE | awk '($3 == "Inode:") {print $4}')`

[ $ABSINO != $REVINO ] && die "inodes differ, my reverse path is bad?"

echo export PATCHSCRIPTS_LIBDIR=$REVERSE

cd $TREE
ln -sf $REVERSE/series/$SERIES series

PATH_ELEMENTS=$(echo $PATH | sed -e 's/:/ /g')

NEW_PATH=$MYDIR/scripts

for p in $PATH_ELEMENTS; do
	if echo $p | grep kernel_patches/scripts > /dev/null 2>&1 ; then
		continue;
	fi
	NEW_PATH="$NEW_PATH:$p"
done

echo export PATH=$NEW_PATH

echo "'$TREE' successfully setup" >&2


#!/bin/sh
#
# CDDL HEADER START
#
# The contents of this file are subject to the terms of the
# Common Development and Distribution License Version 1.0 (CDDL-1.0).
# You can obtain a copy of the license from the top-level file
# "OPENSOLARIS.LICENSE" or at <http://opensource.org/licenses/CDDL-1.0>.
# You may not use this file except in compliance with the license.
#
# CDDL HEADER END
#

#
# Copyright (c) 2018, Intel Corporation.
#

#
# Adjust lustre service degrade state in response to a statechange
#
# ZEVENT_SUBCLASS: 'statechange'
# POOL HEALTH: status from "zpool list health" (either ONLINE or DEGRADED)
#
# depends on lctl(1)
#
# Exit codes:
#   0: normal exit
#   1: lctl missing
#   2: zpool missing
#   3: zfs missing
#   4: Pool status neither "ONLINE" nor "DEGRADED

[ -f "${ZED_ZEDLET_DIR}/zed.rc" ] && . "${ZED_ZEDLET_DIR}/zed.rc"
. "${ZED_ZEDLET_DIR}/zed-functions.sh"

LCTL=${LCTL:-/usr/sbin/lctl}
ZPOOL=${ZPOOL:-/usr/sbin/zpool}
ZFS=${ZFS:-/usr/sbin/zfs}

zed_check_cmd "$LCTL" || exit 1
zed_check_cmd "$ZPOOL" || exit 2
zed_check_cmd "$ZFS" || exit 3

#
# sync_degrade_state (dataset, state)
#
sync_degrade_state()
{
	local dataset="$1"
	local state="$2"
	local service=$($ZFS list -H -o lustre:svname ${dataset})

	zed_log_msg "Lustre:sync_degrade_state pool:${dataset} degraded:${state}"

	if [ -n "${service}" ] && [ "${service}" != "-" ] ; then
		local current=$($LCTL get_param -n obdfilter.${service}.degraded)

		if [ "${current}" != "${state}" ] ; then
			$LCTL set_param obdfilter.${service}.degraded=${state}
		fi
	fi
}

#
# use pool state as deciding factor
#
POOL_STATE=$($ZPOOL list -H -o health ${ZEVENT_POOL})

if [ "${POOL_STATE}" == "ONLINE" ] ; then
	MODE="0"
elif [ "${POOL_STATE}" == "DEGRADED" ] ; then
	MODE="1"
else
	exit 4
fi

#
# visit target pool's datasets and adjust lustre service degrade mode
#
read -r -a DATASETS <<< \
	$($ZFS get -rH -s local -t filesystem -o name lustre:svname ${ZEVENT_POOL})

for dataset in "${DATASETS[@]}" ; do
	sync_degrade_state "${dataset}" "${MODE}"
done

exit 0


#!/bin/sh

# System Profiling Script

TESTS="oprofile iostat vmstat proc_dump"

# common parameters
export OUTPUTDIR=/home/op
export KERNELDIR=/usr/src/linux
export LUSTREDIR=/usr/src/lustre
export PORTALSDIR=/usr/src/portals
NAL=socknal

# Params for OPROFILE
#CTR0_EVENT=CPU_CLK_UNHALTED
CTR0_COUNT=10000

# for intel Petium 4 onwards... Also requires Unit Mask
CTR0_EVENT=GLOBAL_POWER_EVENTS
CTR0_UNIT_MASK=0x01

# Params for VMSTAT
VM_SAMPLING=1

# Params for IOSTAT
IO_SAMPLING=1

# Params for PROC DUMP
export PROC_SAMPLING=2

oprofile_start() {
	echo "$(date +%T): oprofile started...." >>  $OUTPUTDIR/$HOSTNAME/summary
	op_start --ctr0-event=$CTR0_EVENT --ctr0-count=$CTR0_COUNT --ctr0-unit-mask=$CTR0_UNIT_MASK --vmlinux=${KERNELDIR}/vmlinux
	mkdir -p ${OUTPUTDIR}/${HOSTNAME}/oprofile/prof_source/{obdclass,obdecho,osc,ptlrpc,extN,obdfilter,ost,mdc,mds4mds,mds4mds_extN,llite,portals,$NAL}
	mkdir -p ${OUTPUTDIR}/${HOSTNAME}/oprofile/profiling
}

iostat_start() {
	echo "$(date +%T): iostat started...." >>  $OUTPUTDIR/$HOSTNAME/summary
	mkdir ${OUTPUTDIR}/${HOSTNAME}/iostat
	iostat $IO_SAMPLING > $OUTPUTDIR/$HOSTNAME/iostat/iostat.op &
	PID=$!
	echo $PID > $OUTPUTDIR/$HOSTNAME/tmp/iostat.pid
}

vmstat_start() {
	echo $OUTPUTDIR;
	echo "$(date +%T): vmstat started...." >>  $OUTPUTDIR/$HOSTNAME/summary
	mkdir ${OUTPUTDIR}/${HOSTNAME}/vmstat
	vmstat $VM_SAMPLING > $OUTPUTDIR/$HOSTNAME/vmstat/vmstat.op &
	PID=$!
	echo $PID > $OUTPUTDIR/$HOSTNAME/tmp/vmstat.pid
}

proc_dump_start() {
	echo "$(date +%T): proc dump started...." >> $OUTPUTDIR/$HOSTNAME/summary
	sh -c '
		CTRFILE=/$OUTPUTDIR/$HOSTNAME/tmp/running.$$.pid;
		echo $$ > ${CTRFILE};
		while [ -f $CTRFILE ]; do
			cat /proc/meminfo >> $OUTPUTDIR/$HOSTNAME/meminfo;
			cat /proc/interrupts >> $OUTPUTDIR/$HOSTNAME/interrupts;
			cat /proc/net/dev >> $OUTPUTDIR/$HOSTNAME/net-dev;
			sleep ${PROC_SAMPLING};
		done;
		' &
	CTRFILE=/tmp/running.$!.pid
	echo "proc_dump_ctrl $CTRFILE" >> /tmp/prof-ctrl
}

oprofile_stop() {
	op_dump
	op_stop

	echo "$(date +%T): oprofile stopped...." >>  $OUTPUTDIR/$HOSTNAME/summary

	for i in obdclass obdecho osc ptlrpc extN obdfilter mds ost mdc llite
	do
		oprofpp -l ${LUSTREDIR}/${i}/${i}.o >  ${OUTPUTDIR}/${HOSTNAME}/oprofile/profiling/${i}.prof 2>/dev/null
	done

	oprofpp -l ${LUSTREDIR}/mds/mds_extN.o > ${OUTPUTDIR}/${HOSTNAME}/oprofile/profiling/mds_extN.prof 2>/dev/null
	oprofpp -l ${PORTALSDIR}/linux/oslib/portals.o > ${OUTPUTDIR}/${HOSTNAME}/oprofile/profiling/portals.prof 2>/dev/null
	oprofpp -l ${PORTALSDIR}/linux/${NAL}/k${NAL}.o > ${OUTPUTDIR}/${HOSTNAME}/oprofile/profiling/k${NAL}.prof 2>/dev/null

	for i in obdclass obdecho osc ptlrpc extN obdfilter ost mdc llite
	do
		op_to_source --source-dir=${LUSTREDIR}/${i}/ --output-dir=${OUTPUTDIR}/${HOSTNAME}/oprofile/prof_source/${i}/ ${LUSTREDIR}/${i}/${i}.o 2>/dev/null
	done

	op_to_source --source-dir=${LUSTREDIR}/mds/ --output-dir=${OUTPUTDIR}/${HOSTNAME}/oprofile/prof_source/mds4mds/ ${LUSTREDIR}/mds/mds.o 2>/dev/null
	op_to_source --source-dir=${LUSTREDIR}/mds/ --output-dir=${OUTPUTDIR}/${HOSTNAME}/oprofile/prof_source/mds4mds_extN/  ${LUSTREDIR}/mds/mds_extN.o 2>/dev/null

	op_to_source --source-dir=${PORTALSDIR}/linux/oslib/ --output-dir=${OUTPUTDIR}/${HOSTNAME}/oprofile/prof_source/portals ${PORTALSDIR}/linux/oslib/portals.o 2>/dev/null
	op_to_source --source-dir=${PORTALSDIR}/linux/${NAL}/ --output-dir=${OUTPUTDIR}/${HOSTNAME}/oprofile/prof_source/${NAL} ${PORTALSDIR}/linux/${NAL}/k${NAL}.o 2>/dev/null
	op_time -l > ${OUTPUTDIR}/${HOSTNAME}/oprofile/globalprofile 2>/dev/null
}

iostat_stop() {
	echo "$(date +%T): iostat stopped...." >>  $OUTPUTDIR/$HOSTNAME/summary
	PID=$(cat $OUTPUTDIR/$HOSTNAME/tmp/iostat.pid)
	kill $PID
}

vmstat_stop() {
	echo "$(date +%T): vmstat stopped...." >>  $OUTPUTDIR/$HOSTNAME/summary
	PID=$(cat $OUTPUTDIR/$HOSTNAME/tmp/vmstat.pid)
	kill $PID
}

oprofile_dump() {
	op_dump;
}

proc_dump_stop() {
	echo "$(date +%T): proc dump stopped...." >> $OUTPUTDIR/$HOSTNAME/summary
	CTRFILE=$(cat /tmp/prof-ctrl | awk '$1 == "prof_dump_ctrl" {print $2}')
	rm -f $CTRFILE
}

case "$1" in
	start)
		shift;
		while [ ${#*} -gt 1 ]; do
			case "$1" in
				-k)
					shift;
					KERNELDIR=$1;
					;;
				-l)
					shift;
					LUSTREDIR=$1;
					;;
				-p)
					shift;
					PORTALSDIR=$1;
					;;
				-o)
					shift;
					OUTPUTDIR=$1;
					;;
				*)
					echo unrecognized option $1
					break;
					;;
			esac
			shift;
		done

		echo "kerneldir $KERNELDIR" > /tmp/prof-ctrl
		echo -e "\nlustredir $LUSTREDIR" >> /tmp/prof-ctrl
		echo -e "\nportalsdir $PORTALSDIR" >> /tmp/prof-ctrl
		echo -e "\noutputdir $OUTPUTDIR" >> /tmp/prof-ctrl

		if [ -d ${OUTPUTDIR}/${HOSTNAME} ]; then
			echo "Output already exists"
			echo "Please take backup and remove it"
			exit 1
		fi

		mkdir -p ${OUTPUTDIR}/${HOSTNAME}
		echo -e "Profiling started on $HOSTNAME" >  ${OUTPUTDIR}/${HOSTNAME}/summary
		echo -e "\n\nModules Listing on $HOSTNAME" >> ${OUTPUTDIR}/${HOSTNAME}/summary
		/sbin/lsmod >> ${OUTPUTDIR}/${HOSTNAME}/summary
		echo -e "\n\nKernel : " >> ${OUTPUTDIR}/${HOSTNAME}/summary
		uname -a  >> ${OUTPUTDIR}/${HOSTNAME}/summary
		echo -e "\n\nPCI Devices : "  >> ${OUTPUTDIR}/${HOSTNAME}/summary
		lspci -t -v  >> ${OUTPUTDIR}/${HOSTNAME}/summary
		echo -e "\n\nTests carried out " >> ${OUTPUTDIR}/${HOSTNAME}/summary
		mkdir $OUTPUTDIR/$HOSTNAME/tmp

		for test in $TESTS; do
			${test}_start;
		done
	;;
	stop)
		KERNELDIR=$(cat /tmp/prof-ctrl | awk '$1 == "kerneldir" {print $2}')
		LUSTREDIR=$(cat /tmp/prof-ctrl | awk '$1 == "lustredir" {print $2}')
		PORTALSDIR=$(cat /tmp/prof-ctrl | awk '$1 == "portalsdir" {print $2}')
		OUTPUTDIR=$(cat /tmp/prof-ctrl | awk '$1 == "outputdir" {print $2}')

		for test in $TESTS; do
			${test}_stop;
		done

		rm -rf ${OUTPUTDIR}/$HOSTNAME/tmp
		tar -cf ${OUTPUTDIR}/${HOSTNAME}.tar  ${OUTPUTDIR}/${HOSTNAME}
		echo "Dumped results in ${OUTPUTDIR}/${HOSTNAME}.tar"
	;;
	dump)
		if "oprofile" in $TESTS; then
			oprofile_dump;
		fi
	;;
	clean)
		OUTPUTDIR=$(cat /tmp/prof-ctrl | awk '$1 == "outputdir" {print $2}')

		echo Deleting directory $OUTPUTDIR/$HOSTNAME ...
		rm -rf $OUTPUTDIR/$HOSTNAME
		echo Deleting file $OUTPUTDIR/${HOSTNAME}.tar ...
		rm -rf $OUTPUTDIR/${HOSTNAME}.tar
		echo Deleting oprofile samples ...
		rm -f /var/lib/oprofile/samples/*
		rm -f /tmp/prof-ctrl
	;;
	*)
		echo $"Usage : $0 {start|stop|dump|clean} [OPTIONS]";
		echo $"OPTIONS :"
		echo $" -l lustre_dir"
		echo $" -p portals_dir"
		echo $" -k kernel_dir"
		echo $" -o output_dir"
		exit 1
esac

exit 0


#!/bin/bash
#
# usage: acceptance-small.sh [test list]
#	if no tests are specified, they are taken from test-groups/regression
#	if {TEST_NAME}=no is set, that test script is skipped
DEFAULT_SUITES="${@:-$ACC_SM_ONLY}"
DEFAULT_SUITES="${DEFAULT_SUITES:-$(cat $LUSTRE/tests/test-groups/regression)}"
for SUB in $DEFAULT_SUITES; do
	ENV=$(echo $SUB | tr "[:lower:]-" "[:upper:]_")
	[ "$(eval echo \$$ENV)" = "no" ] && continue
	SUITES="$SUITES $SUB"
done
sh auster -r -R -v -f ${NAME:-lustre} $SUITES


#!/bin/sh
set -evx

MOUNT=${MOUNT:-/mnt/lustre}
DIR=${DIR:-$MOUNT}
SRC=${SRC:-`dirname $0`/../..}
export CC=${CC:-gcc}
while date; do
	for i in lustre; do
		TGT=$DIR/$i
		[ -d $TGT ] || cp -av $SRC/$i/ $TGT
		make -C $TGT clean
		make -C $TGT -j2
		make -C $TGT clean
	done
done


#!/bin/bash
#
# run tests on different directories at the same time
#

set -e

SRCDIR=$(cd $(dirname $0); echo $PWD)
PATH=$PWD/$SRCDIR:$SRCDIR:$SRCDIR/../utils:$PATH

SAVE_PWD=$PWD

LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
. $LUSTRE/tests/test-framework.sh
init_test_env $@
. ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
init_logging

PARALLEL_RUNS=${PARALLEL_RUNS:-2}
FAIL_ON_ERROR=false

check_and_setup_lustre

build_test_filter

DIR=${DIR:-$MOUNT}

ORIGIN_DIR=$DIR

prepare_running_directories()
{
	local mdtidx
	local rc=0
	local i

	for i in $(seq $PARALLEL_RUNS); do
		rm -rf $ORIGIN_DIR/dir$i
		if [ $MDSCOUNT -gt 1 ]; then
			mdtidx=$((i % MDSCOUNT))
			$LFS mkdir -i $mdtidx $ORIGIN_DIR/dir$i || rc=$?
		else
			mkdir -p $ORIGIN_DIR/dir$i
		fi

		if [ $rc != 0 ]; then
			echo "can not create dir$i"
			break
		fi
	done

	return $rc
}

prepare_running_directories || error "Can not create running directories"

cleanup_running_directories()
{
	local i
	local rc=0

	for i in $(seq $PARALLEL_RUNS); do
		rm -rf $ORIGIN_DIR/dir$i || rc=$?
	done

	return $rc
}

test_sanity()
{
	[ ! -f sanity.sh ] && skip_env "No sanity.sh skipping"

	local index
	local pid
	local rpids
	local rc=0
	local rrc=0

	for index in $(seq $PARALLEL_RUNS); do
		DIR=$ORIGIN_DIR/dir${index} PARALLEL=yes \
		EXT2_DEV="$TMP/SANITY.LOOP_${index}" \
		LOGDIR="${LOGDIR}_${index}" YAML_LOG="" sh sanity.sh &
		pid=$!
		echo start sanity: $pid
		rpids="$rpids $pid"
	done

	echo sanity pids: $rpids
	for pid in $rpids; do
		wait $pid
		rc=$?
		echo "pid=$pid rc=$rc"
		if [ $rc != 0 ]; then
			rrc=$((rrc + 1))
		fi
	done
	return $rrc
}

run_test sanity "Run sanity parallel on different directories at the same time"

cleanup_running_directories || error "unlink running directories"

complete $SECONDS
check_and_cleanup_lustre
exit_status


#!/bin/bash
#
# Run select tests by setting ONLY, or as arguments to the script.
# Skip specific tests by setting EXCEPT.
#

set -e

ONLY=${ONLY:-"$*"}
ALWAYS_EXCEPT=${ALWAYS_EXCEPT:-"$DOM_PERFORMANCE_EXCEPT"}
# UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!

[ "$SLOW" = "no" ] && EXCEPT_SLOW=""

LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}

. $LUSTRE/tests/test-framework.sh

init_test_env $@
. ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
init_logging

SAVED_FAIL_ON_ERROR=$FAIL_ON_ERROR
FAIL_ON_ERROR=false

SAVED_DEBUG=$($LCTL get_param -n debug 2> /dev/null)

. $LUSTRE/tests/functions.sh
build_test_filter
check_and_setup_lustre

# if MACHINEFILE set and exists -- use it
MACHINEFILE=${MACHINEFILE:-$TMP/$(basename $0 .sh)-$(hostname).machines}
clients=${CLIENTS:-$HOSTNAME}
generate_machine_file $clients $MACHINEFILE ||
	error "Failed to generate machine file"

DP_DIO=${DP_DIO:-"no"}

DOM_SIZE=${DOM_SIZE:-"1M"}
DP_OSC="mdc"

DP_NORM=$DIR/dp_norm
DP_DOM=$DIR/dp_dom
DP_DOM_DNE=$DIR/dp_dne
DP_STATS=${DP_STATS:-"no"}

if $DO_CLEANUP; then
	rm -rf $DIR/*
else
	rm -rf $DP_NORM $DP_DOM $DP_DOM_DNE
fi

# total number of files
DP_FNUM=${DP_FNUM:-16384}
# number of threads
DP_NUM=${DP_NUM:-4}

# 1 stripe for normal files
mkdir -p $DP_NORM
$LFS setstripe -c 2 $DP_NORM ||
	error "Cannot create test directory for ordinary files"

if [[ $MDSCOUNT -gt 1 ]] ; then
	$LFS setdirstripe -i 0 -c $MDSCOUNT $DP_DOM_DNE ||
		error_noexit "Cannot create striped directory"
	$LFS setstripe -E ${DOM_SIZE} -L mdt -E EOF $DP_DOM_DNE ||
		error_noexit "Cannot create test directory for dom files"
fi

mkdir -p $DP_DOM
$LFS setstripe -E ${DOM_SIZE} -L mdt -E EOF $DP_DOM ||
	error "Cannot create test directory for dom files"

dp_clear_stats() {
	local cli=$1

	$LCTL set_param -n osc.*.stats=0
	$LCTL set_param -n mdc.*.stats=0
	$LCTL set_param -n ${cli}.*.${cli}_stats=0
	$LCTL set_param -n ${cli}.*.rpc_stats=0
	$LCTL set_param -n llite.*.read_ahead_stats=0
	$LCTL set_param -n llite.*.unstable_stats=0
}

dp_collect_stats() {
	local cli=$1

	sync;sync
	echo ----- MDC RPCs: $(calc_stats mdc.*.stats req_active)
	echo ----- OSC RPCs: $(calc_stats osc.*.stats req_active)

	if [ "x$DP_STATS" != "xyes" ] ; then
		return 0
	fi

	$LCTL get_param ${cli}.*.${cli}_stats
	$LCTL get_param ${cli}.*.rpc_stats
	# for OSC get both OSC and MDC stats
	if [ $cli == "osc" ] ; then
		$LCTL get_param mdc.*.stats
	fi
	$LCTL get_param ${cli}.*.stats
	$LCTL get_param ${cli}.*.unstable_stats
	$LCTL get_param ${cli}.*.${cli}_cached_mb
	$LCTL get_param llite.*.read_ahead_stats
}

dp_setup_test() {
	local cli=$1

	cancel_lru_locks $cli
	### drop all debug except critical
	$LCTL set_param -n debug="error warning console emerg"
	dp_clear_stats $cli
}

dp_run_cmd() {
	local cmd=$1
	local cmdlog=$TMP/dp_cmd.log
	local rc

	dp_setup_test $DP_OSC
	if ! grep -qw "$MOUNT" /proc/mounts ; then
		echo "!!!!! Lustre is not mounted !!!!!, aborting"
		return 0
	fi

	echo "## $cmd" | awk '{ if (NR==1) {gsub(/[ \t\r\n]+/, " "); \
				gsub(/\|.*$/, ""); print }}'
	echo "## $(date +'%F %H:%M:%S'): START"
	eval $cmd 2>&1 | tee $cmdlog || true

	rc=${PIPESTATUS[0]}
	if [ $rc -eq 0 ] && grep -q "p4_error:" $cmdlog ; then
		rc=1
	fi

	dp_collect_stats $DP_OSC
	remount_client $DIR > /dev/null
	return $rc
}

run_MDtest() {
	if ! which mdtest > /dev/null 2>&1 ; then
		skip_env "Mdtest is not installed, skipping"
	fi

	local mdtest=$(which mdtest)

	local TDIR=${1:-$MOUNT}
	local th_num=$((DP_FNUM * 2 / DP_NUM))
	local bsizes="8192"

	chmod 0777 $TDIR

	[ "$SLOW" = "yes" ] && bsizes="4096 32768"

	for bsize in $bsizes ; do
		dp_run_cmd "mpi_run -np $DP_NUM $mdtest -i 3 -I $th_num -F \
			-z 1 -b 1 -L -u -w $bsize -R -d $TDIR"
		if [ ${PIPESTATUS[0]} != 0 ]; then
			error "MDtest failed, aborting"
		fi
	done

	rm -rf $TDIR/*
	return 0
}

run_SmallIO() {
	local TDIR=${1:-$DIR}
	local count=$DP_FNUM

	local MIN=$((count * 16))
	local mdssize=$(mdssize_from_index $TDIR 0)
	[ $mdssize -le $MIN ] && count=$((mdssize / 16))

	dp_run_cmd "createmany -o $TDIR/file- $count | grep 'total:'"
	if [ ${PIPESTATUS[0]} != 0 ]; then
		error "File creation failed, aborting"
	fi

	dp_run_cmd "statmany -s $TDIR/file- $count $((count * 5)) |
		grep 'total:'"
	if [ ${PIPESTATUS[0]} != 0 ]; then
		error "File stat failed, aborting"
	fi

	for opc in w a r ; do
		dp_run_cmd "smalliomany -${opc} $TDIR/file- $count 300 |
			grep 'total:'"
		if [ ${PIPESTATUS[0]} != 0 ]; then
			error "SmallIO -${opc} failed, aborting"
		fi

	done

	dp_run_cmd "unlinkmany $TDIR/file- $count | grep 'total:'"
	if [ ${PIPESTATUS[0]} != 0 ]; then
		error "SmallIO failed, aborting"
	fi

	return 0
}

run_IOR() {
	if ! which IOR > /dev/null 2>&1 ; then
		skip_env "IOR is not installed, skipping"
	fi

	local IOR=$(which IOR)
	local iter=$((DP_FNUM / DP_NUM))
	local direct=""

	if [ "x$DP_DIO" == "xyes" ] ; then
		direct="-B"
	fi

	local TDIR=${1:-$MOUNT}

	chmod 0777 $TDIR

	# for DoM large files (beyond the DoM size) use
	# DOM_SIZE=1M :
	#     bsize="4096 " - 4Mb
	#     nsegments=$((128 * 1024))
	# DOM_SIZE=64k :
	#     bsize="1024 " - 1Mb
	#     nsegments=$((32 * 1024))
	local bsizes=${BSIZES:-"4 32"}
	local nsegments=${NSEGMENTS:-128}
	[ "$SLOW" = "no" ] && bsizes="8"

	for bsize in $bsizes ; do
		segments=$((nsegments / bsize))

		dp_run_cmd "mpi_run -np $DP_NUM $IOR \
			-a POSIX -b ${bsize}K -t ${bsize}K -o $TDIR/ -k \
			-s $segments -w -r -i $iter -F -E -z -m -Z $direct" |
			awk '($1 !~ /^(write|read|access)$/) || NF>12 {print}'
		if [ ${PIPESTATUS[0]} != 0 ]; then
			error "IOR write test for ${bsize}K failed, aborting"
		fi

		# check READ performance only (no cache)
		dp_run_cmd "mpi_run -np $DP_NUM $IOR \
			-a POSIX -b ${bsize}K -t ${bsize}K -o $TDIR/ -X 42\
			-s $segments -r -i $iter -F -E -z -m -Z $direct" |
			awk '($1 !~ /^(read|access|remove)$/) || NF>12 {print}'
		if [ ${PIPESTATUS[0]} != 0 ]; then
			error "IOR read test for ${bsize}K failed, aborting"
		fi

	done
	rm -rf $TDIR/*
	return 0
}

run_Dbench() {
	if ! which dbench > /dev/null 2>&1 ; then
		skip_env "Dbench is not installed, skipping"
	fi

	local TDIR=${1:-$MOUNT}

	if [ "x$DP_DOM_DNE" == "x$TDIR" ] ; then
		echo "dbench uses subdirs, skipping for DNE dir"
		return 0
	fi

	dp_run_cmd "dbench -D $TDIR $DP_NUM | egrep -v 'warmup|execute'"
	if [ ${PIPESTATUS[0]} != 0 ]; then
		error "Dbench failed, aborting"
	fi

	rm -rf $TDIR/*
	return 0
}

run_FIO() {
	# https://github.com/axboe/fio/archive/fio-2.8.zip
	if ! which fio > /dev/null 2>&1 ; then
		skip_env "No FIO installed, skipping"
	fi

	local fnum=128 # per thread
	local total=$((fnum * DP_NUM)) # files in all threads
	local loops=$((DP_FNUM / total)) # number of loops
	local direct=""
	local output=""

	if [ $loops -eq 0 ] ; then
		loops=1
	fi

	if [ "x$DP_DIO" == "xyes" ] ; then
		direct="--direct=1"
	else
		direct="--buffered=1 --bs_unaligned=1"
	fi

	if [ "x$DP_STATS" != "xyes" ] ; then
		output="--minimal"
	fi

	local TDIR=${1:-$MOUNT}
	base_cmd="fio --name=smallio --ioengine=posixaio $output \
		  --iodepth=$((DP_NUM * 4)) --directory=$TDIR \
		  --nrfiles=$fnum --openfiles=10000 \
		  --numjobs=$DP_NUM --filesize=64k --lockfile=readwrite"

	dp_run_cmd "$base_cmd --create_only=1" > /dev/null
	if [ ${PIPESTATUS[0]} != 0 ]; then
		error "FIO file creation failed, aborting"
	fi

	local bsizes="8"
	[ "$SLOW" = "yes" ] && bsizes="4 32"

	for bsize in $bsizes ; do
		local write_cmd="$base_cmd --bs=${bsize}k --rw=randwrite \
			$direct --file_service_type=random --randrepeat=1 \
			 --norandommap --group_reporting=1 --loops=$loops"
		if [ "x$DP_STATS" != "xyes" ] ; then
			dp_run_cmd "$write_cmd | awk -F\; '{printf \"WRITE: \
				BW %dKiB/sec, IOPS %d, lat (%d/%d/%d)usec\n\", \
				\$48, \$49, \$53, \$57, \$81}'"
		else
			dp_run_cmd "$write_cmd"
		fi
		if [ ${PIPESTATUS[0]} != 0 ]; then
			error "FIO write test with ${bsize}k failed, aborting"
		fi

		local read_cmd="$base_cmd --bs=${bsize}k --rw=randread \
			$direct --file_service_type=random --randrepeat=1 \
			 --norandommap --group_reporting=1 --loops=$loops"
		if [ "x$DP_STATS" != "xyes" ] ; then
			dp_run_cmd "$read_cmd | awk -F\; '{printf \"READ : \
				BW %dKiB/sec, IOPS %d, lat (%d/%d/%d)usec\n\", \
				\$7, \$8, \$12, \$16, \$40}'"
		else
			dp_run_cmd "$read_cmd"
		fi
		if [ ${PIPESTATUS[0]} != 0 ]; then
			error "FIO read test with ${bsize}k failed, aborting"
		fi
	done
	rm -rf $TDIR/*
	return 0
}

run_compbench() {
	local compilebench
	if [ x$cbench_DIR = x ]; then
		compilebench=$(which compilebench 2> /dev/null)
	else
		cd $cbench_DIR
		[ -x compilebench ] ||
			skip_env "compilebench is missing in $cbench_DIR"
		compilebench=compilebench
	fi

	[ x$compilebench != x ] ||
		skip_env "Compilebench is not installed, skipping"

	local TDIR=${1:-$MOUNT}

	dp_run_cmd "$compilebench -D $TDIR -i 2 -r 2 --makej"
	if [ ${PIPESTATUS[0]} != 0 ]; then
		error "Compilebench failed, aborting"
	fi

	rm -rf $TDIR/*
}

dp_test_run() {
	local test=$1
	local facets=$(get_facets MDS)
	local nodes=$(comma_list $(mdts_nodes))
	local p="$TMP/$TESTSUITE-$TESTNAME.parameters"

	save_lustre_params $facets "mdt.*.dom_lock" >> $p

	printf "\n##### $test: DoM files\n"
	do_nodes $nodes "lctl set_param -n mdt.*.dom_lock=1"
	DP_OSC="mdc"
	run_${test} $DP_DOM

	if [ -d $DP_DOM_DNE ] ; then
		printf "\n##### $test: DoM files + DNE\n"
		DP_OSC="mdc"
		run_${test} $DP_DOM_DNE
	fi

	printf "\n##### $test: OST files\n"
	DP_OSC="osc"
	run_${test} $DP_NORM

	restore_lustre_params < $p
	rm -f $p
}

test_smallio() {
	dp_test_run SmallIO
}
run_test smallio "Performance comparision: smallio"

test_mdtest() {
	dp_test_run MDtest
}
run_test mdtest "Performance comparision: mdtest"

test_IOR() {
	dp_test_run IOR
}
run_test IOR "Performance comparision: IOR"

test_dbench() {
	dp_test_run Dbench
}
run_test dbench "Performance comparision: dbench"

test_fio() {
	dp_test_run FIO
}
run_test fio "Performance comparision: FIO"

test_compbench() {
	dp_test_run compbench
}
run_test compbench "Performance comparision: compilebench"

FAIL_ON_ERROR=$SAVED_FAIL_ON_ERROR
$LCTL set_param -n debug="$SAVED_DEBUG"

complete $SECONDS
check_and_cleanup_lustre
exit_status


#!/bin/bash

# Simple function used by run_*.sh scripts

assert_env() {
    local failed=""
    for name in $@; do
        if [ -z "${!name}" ]; then
            echo "$0: $name must be set"
            failed=1
        fi
    done
    [ $failed ] && exit 1 || true
}

# lrepl - Lustre test Read-Eval-Print Loop.
#
# This function implements a REPL for the Lustre test framework.  It
# doesn't exec an actual shell because the user may want to inspect
# variables and use functions from the test framework.
lrepl() {
    local line
    local rawline
    local prompt

    cat <<EOF
        This is an interactive read-eval-print loop interactive shell
        simulation that you can use to debug failing tests.  You can
        enter most bash command lines (see notes below).

        Use this REPL to inspect variables, set them, call test
        framework shell functions, etcetera.

        'exit' or EOF to exit this shell.

        set \$retcode to 0 to cause the assertion failure that
        triggered this REPL to be ignored.

        Examples:
            do_facet ost1 lctl get_param ost.*.ost.threads_*
            do_rpc_nodes \$OSTNODES unload_modules

        NOTES:
            All but the last line of multi-line statements or blocks
            must end in a backslash.

            "Here documents" are not supported.

            History is not supported, but command-line editing is.

EOF

    # Prompt escapes don't work in read -p, sadly.
    prompt=":${TESTNAME:-UNKNOWN}:$(uname -n):$(basename $PWD)% "

    # We use read -r to get close to a shell experience
    while read -e -r -p "$prompt" rawline; do
        line=
        case "$rawline" in
        # Don't want to exit-exit, just exit the REPL
        exit) break;;
        # We need to handle continuations, and read -r doesn't do
        # that for us.  Yet we need read -r.
        #
        # We also use case/esac to compare lines read to "*\\"
        # because [ "$line" = *\\ ] and variants of that don't work.
        *\\) line="$rawline"
            while read -e -r -p '> ' rawline
            do
                line="$line"$'\n'"$rawline"
                case "$rawline" in
                # We could check for here documents by matching
                # against *<<*, but who cares.
                *\\) continue;;
                *) break;;
                esac
            done
            ;;
        *) line=$rawline
        esac

        case "$line" in
        *\\) break;;
        esac

        # Finally!  Time to eval.
        eval "$line"
    done

    echo $'\n\tExiting interactive shell...\n'
    return 0
}

# lassert - Lustre test framework assert
#
# Arguments: failure code, failure message, expression/statement
#
# lassert evaluates the expression given, and, if false, calls
# error() to trigger test failure.  If REPL_ON_LASSERT is true then
# lassert will call lrepl() to give the user an interactive shell.
# If the REPL sets retcode=0 then the assertion failure will be
# ignored.
lassert() {
    local retcode=$1
    local msg=$2
    shift 2

    echo "checking $* ($(eval echo \""$*"\"))..."
    eval "$@" && return 0;

    if ${REPL_ON_LASSERT:-false}; then
        echo "Assertion $retcode failed: $* (expanded: $(eval echo \""$*"\"))
$msg"
        lrepl
    fi

    error "Assertion $retcode failed: $* (expanded: $(eval echo \""$*"\"))
$msg"
    return $retcode
}

# setmodopts- set module options for subsequent calls to load_modules
#
# Usage: setmodopts module_name new_value [var_in_which_to_save_old_value]
#        setmodopts -a module_name new_value [var_in_which_to_save_old_value]
#
# In the second usage the new value is appended to the old.
setmodopts() {
        local _append=false

        if [ "$1" = -a ]; then
            _append=true
            shift
        fi

        local _var=MODOPTS_$1
        local _newvalue=$2
        local _savevar=$3
        local _oldvalue

        # Dynamic naming of variables is a pain in bash.  In ksh93 we could
        # write "nameref opts_var=${modname}_MODOPTS" then assign directly
        # to opts_var.  Associative arrays would also help, alternatively.
        # Alas, we're stuck with eval until all distros move to a more recent
        # version of bash.  Fortunately we don't need to eval unset and export.

        if [ -z "$_newvalue" ]; then
            unset $_var
            return 0
        fi

        _oldvalue=${!var}
        $_append && _newvalue="$_oldvalue $_newvalue"
        export $_var="$_newvalue"
        echo setmodopts: ${_var}=${_newvalue}

        [ -n "$_savevar" ] && eval $_savevar=\""$_oldvalue"\"
}

echoerr () { echo "$@" 1>&2 ; }

signaled() {
    echoerr "$(date +'%F %H:%M:%S'): client load was signaled to terminate"

    local PGID=$(ps -eo "%c %p %r" | awk "/ $PPID / {print \$3}")
    kill -TERM -$PGID
    sleep 5
    kill -KILL -$PGID
}

mpi_run () {
    local mpirun="$MPIRUN $MPIRUN_OPTIONS --oversubscribe"
    local command="$mpirun $@"
    local mpilog=$TMP/mpi.log
    local rc

    if [ -n "$MPI_USER" -a "$MPI_USER" != root -a -n "$mpirun" ]; then
        echo "+ chmod 0777 $MOUNT"
        chmod 0777 $MOUNT
        command="su $MPI_USER sh -c \"$command \""
    fi

    ls -ald $MOUNT
    echo "+ $command"
    eval $command 2>&1 | tee $mpilog || true

    rc=${PIPESTATUS[0]}
    if [ $rc -eq 0 ] && grep -q "p4_error:" $mpilog ; then
       rc=1
    fi
    return $rc
}

nids_list () {
	local list
	local escape="$2"
	for i in ${1//,/ }; do
		if [ "$list" = "" ]; then
			list="$i@$NETTYPE"
		else
			list="$list$escape $i@$NETTYPE"
		fi
	done
	echo $list
}

# FIXME: all setup/cleanup can be done without rpc.sh
lst_end_session () {
    local verbose=false
    [ x$1 = x--verbose ] && verbose=true

    export LST_SESSION=`$LST show_session 2>/dev/null | awk -F " " '{print $5}'`
    [ "$LST_SESSION" == "" ] && return

	$LST stop b
    if $verbose; then
        $LST show_error c s
    fi
    $LST end_session
}

lst_session_cleanup_all () {
    local list=$(comma_list $(nodes_list))
    do_rpc_nodes $list lst_end_session
}

lst_cleanup () {
    lsmod | grep -q lnet_selftest && \
        rmmod lnet_selftest > /dev/null 2>&1 || true
}

lst_cleanup_all () {
   local list=$(comma_list $(nodes_list))

   # lst end_session needs to be executed only locally
   # i.e. on node where lst new_session was called
   lst_end_session --verbose
   do_rpc_nodes $list lst_cleanup
}

lst_setup () {
    load_module lnet_selftest
}

lst_setup_all () {
    local list=$(comma_list $(nodes_list))
    do_rpc_nodes $list lst_setup
}

###
# short_hostname
#
# Passed a single argument, strips everything off following
# and includes the first period.
# client-20.lab.whamcloud.com becomes client-20
short_hostname() {
  echo $(sed 's/\..*//' <<< $1)
}

###
# short_nodename
#
# Find remote nodename, stripped of any domain, etc.
# 'hostname -s' is easy, but not implemented on all systems
short_nodename() {
	local rname=$(do_node $1 "uname -n" || echo -1)
	if [[ "$rname" = "-1" ]]; then
		rname=$1
	fi
	echo $(short_hostname $rname)
}

print_opts () {
    local var

    echo OPTIONS:

    for i in $@; do
        var=$i
        echo "${var}=${!var}"
    done
    [ -e $MACHINEFILE ] && cat $MACHINEFILE
}

is_lustre () {
	[ "$(stat -f -c %T $1)" = "lustre" ]
}

setstripe_getstripe () {
	local file=$1
	shift
	local params=$@

	is_lustre $file || return 0

	if [ -n "$params" ]; then
		$LFS setstripe $params $file ||
			error "setstripe $params failed"
	fi
	$LFS getstripe $file ||
		error "getstripe $file failed"
}

run_compilebench() {
	local dir=${1:-$DIR}
	local cbench_DIR=${cbench_DIR:-""}
	local cbench_IDIRS=${cbench_IDIRS:-2}
	local cbench_RUNS=${cbench_RUNS:-2}

	print_opts cbench_DIR cbench_IDIRS cbench_RUNS

	[ x$cbench_DIR = x ] &&
		skip_env "compilebench not found"

	[ -e $cbench_DIR/compilebench ] ||
		skip_env "No compilebench build"

	# Space estimation:
	# compile dir kernel-0	~1GB
	# required space	~1GB * cbench_IDIRS
	local space=$(df -P $dir | tail -n 1 | awk '{ print $4 }')
	if [[ $space -le $((1024 * 1024 * cbench_IDIRS)) ]]; then
		cbench_IDIRS=$((space / 1024 / 1024))
		[[ $cbench_IDIRS -eq 0 ]] &&
			skip_env "Need free space at least 1GB, have $space"

		echo "reducing initial dirs to $cbench_IDIRS"
	fi
	echo "free space = $space KB"

	# FIXME:
	# t-f _base needs to be modifyed to set properly tdir
	# for new "test_foo" functions names
	# local testdir=$DIR/$tdir
	local testdir=$dir/d0.compilebench.$$
	test_mkdir -p $testdir
	setstripe_getstripe $testdir $cbench_STRIPEPARAMS

    local savePWD=$PWD
    cd $cbench_DIR
    local cmd="./compilebench -D $testdir -i $cbench_IDIRS \
        -r $cbench_RUNS --makej"

    log "$cmd"

    local rc=0
    eval $cmd
    rc=$?

    cd $savePWD
    [ $rc = 0 ] || error "compilebench failed: $rc"
    rm -rf $testdir
}

run_metabench() {
	local dir=${1:-$DIR}
	local mntpt=${2:-$MOUNT}
	METABENCH=${METABENCH:-$(which metabench 2> /dev/null || true)}
	mbench_NFILES=${mbench_NFILES:-30400}
	# threads per client
	mbench_THREADS=${mbench_THREADS:-4}
	mbench_OPTIONS=${mbench_OPTIONS:-}
	mbench_CLEANUP=${mbench_CLEANUP:-true}

	[ x$METABENCH = x ] && skip_env "metabench not found"

	print_opts METABENCH clients mbench_NFILES mbench_THREADS

	local testdir=$dir/d0.metabench
	test_mkdir -p $testdir
	setstripe_getstripe $testdir $mbench_STRIPEPARAMS

	# mpi_run uses mpiuser
	chmod 0777 $testdir

	# -C             Run the file creation tests. Creates zero byte files.
	# -S             Run the file stat tests.
	# -c nfile       Number of files to be used in each test.
	# -k             Cleanup files when finished.
	local cmd="$METABENCH -w $testdir -c $mbench_NFILES -C -S $mbench_OPTIONS"
	echo "+ $cmd"

	# find out if we need to use srun by checking $SRUN_PARTITION
	if [ "$SRUN_PARTITION" ]; then
		$SRUN $SRUN_OPTIONS -D $testdir -w $clients -N $num_clients \
			-n $((num_clients * mbench_THREADS)) \
			-p $SRUN_PARTITION -- $cmd
	else
		mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
			-np $((num_clients * $mbench_THREADS)) $cmd
	fi

	local rc=$?
	if [ $rc != 0 ] ; then
		error "metabench failed! $rc"
	fi

	if $mbench_CLEANUP; then
		rm -rf $testdir
	else
		mv $dir/d0.metabench $mntpt/_xxx.$(date +%s).d0.metabench
	fi
}

run_simul() {
	SIMUL=${SIMUL:=$(which simul 2> /dev/null || true)}
	[ x$SIMUL = x ] && skip_env "simul not found"
	[ "$NFSCLIENT" ] && skip "skipped for NFSCLIENT mode"

	# threads per client
	simul_THREADS=${simul_THREADS:-2}
	simul_REP=${simul_REP:-20}

	# FIXME
	# Need space estimation here.

	print_opts SIMUL clients simul_REP simul_THREADS

	local testdir=$DIR/d0.simul
	test_mkdir $testdir
	setstripe_getstripe $testdir $simul_STRIPEPARAMS

	# mpi_run uses mpiuser
	chmod 0777 $testdir

	# -n # : repeat each test # times
	# -N # : repeat the entire set of tests # times

	local cmd="$SIMUL -d $testdir -n $simul_REP -N $simul_REP"

	echo "+ $cmd"
	# find out if we need to use srun by checking $SRUN_PARTITION
	if [ "$SRUN_PARTITION" ]; then
		$SRUN $SRUN_OPTIONS -D $testdir -w $clients -N $num_clients \
			-n $((num_clients * simul_THREADS)) -p $SRUN_PARTITION \
			-- $cmd
	else
		mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
			-np $((num_clients * simul_THREADS)) $cmd
	fi

    local rc=$?
    if [ $rc != 0 ] ; then
        error "simul failed! $rc"
    fi
    rm -rf $testdir
}

run_mdtest() {
	MDTEST=${MDTEST:=$(which mdtest 2> /dev/null || true)}
	[ x$MDTEST = x ] && skip_env "mdtest not found"
	[ "$NFSCLIENT" ] && skip "skipped for NFSCLIENT mode"

	# threads per client
	mdtest_THREADS=${mdtest_THREADS:-2}
	mdtest_nFiles=${mdtest_nFiles:-"100000"}
	# We devide the files by number of core
	mdtest_nFiles=$((mdtest_nFiles/mdtest_THREADS/num_clients))
	mdtest_iteration=${mdtest_iteration:-1}
	local mdtest_custom_params=${mdtest_custom_params:-""}
	local type=${1:-"ssf"}

	local mdtest_Nmntp=${mdtest_Nmntp:-1}

	if [ $type = "ssf" ] && [ $mdtest_Nmntp -ne 1 ]; then
		skip "shared directory mode is not compatible" \
			"with multiple directory paths"
	fi

	# FIXME
	# Need space estimation here.

	print_opts MDTEST mdtest_iteration mdtest_THREADS mdtest_nFiles

	local testdir=$DIR/d0.mdtest
	test_mkdir $testdir
	setstripe_getstripe $testdir $mdtest_STRIPEPARAMS
	chmod 0777 $testdir

	for ((i=1; i<mdtest_Nmntp; i++)); do
		zconf_mount_clients $clients $MOUNT$i "$mntopts" ||
			error_exit "Failed $clients on $MOUNT$i"
		local dir=$DIR$i/d0.mdtest$i
		test_mkdir $dir
		setstripe_getstripe $dir $mdtest_SETSTRIPEPARAMS
		chmod 0777 $dir
		testdir="$testdir@$dir"
	done
	# mpi_run uses mpiuser

	# -i # : repeat each test # times
	# -d   : test dir
	# -n # : number of file/dir to create/stat/remove
	# -u   : each process create/stat/remove individually

	local cmd="$MDTEST -d $testdir -i $mdtest_iteration \
		-n $mdtest_nFiles $mdtest_custom_params"

	[ $type = "fpp" ] && cmd="$cmd -u"

	echo "+ $cmd"
	# find out if we need to use srun by checking $SRUN_PARTITION
	if [ "$SRUN_PARTITION" ]; then
		$SRUN $SRUN_OPTIONS -D $testdir -w $clients -N $num_clients \
			-n $((num_clients * mdtest_THREADS)) \
			-p $SRUN_PARTITION -- $cmd
	else
		mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
			-np $((num_clients * mdtest_THREADS)) $cmd
	fi

    local rc=$?
    if [ $rc != 0 ] ; then
        error "mdtest failed! $rc"
    fi
	rm -rf $testdir
	for ((i=1; i<mdtest_Nmntp; i++)); do
		local dir=$DIR$i/d0.mdtest$i
		rm -rf $dir
		zconf_umount_clients $clients $MOUNT$i ||
			error_exit "Failed umount $MOUNT$i on $clients"
	done
}

run_connectathon() {
	local dir=${1:-$DIR}
	cnt_DIR=${cnt_DIR:-""}
	cnt_NRUN=${cnt_NRUN:-10}

	print_opts cnt_DIR cnt_NRUN

	[ x$cnt_DIR = x ] && skip_env "connectathon dir not found"
	[ -e $cnt_DIR/runtests ] || skip_env "No connectathon runtests found"

	# Space estimation:
	# "special" tests create a 30 MB file + misc. small files
	# required space ~40 MB
	local space=$(df -P $dir | tail -n 1 | awk '{ print $4 }')
	if [[ $space -le $((1024 * 40)) ]]; then
		skip_env "Need free space at least 40MB, have $space KB"
	fi
	echo "free space = $space KB"

	local testdir=$dir/d0.connectathon
	test_mkdir -p $testdir
	setstripe_getstripe $testdir $cnt_STRIPEPARAMS

	local savePWD=$PWD
	cd $cnt_DIR

	#
	# To run connectathon:
	# runtests [-a|-b|-g|-s|-l] [-f|-n|-t] [-N numpasses] [test-directory]
	#
	# One of the following test types
	#    -b  basic
	#    -g  general
	#    -s  special
	#    -l  lock
	#    -a  all of the above
	#
	# -f  a quick functional test
	# -n  suppress directory operations (mkdir and rmdir)
	# -t  run with time statistics (default for basic tests)
	#
	# -N numpasses - specifies the number of times to run
	#                the tests. Optional.

	tests="-b -g -s"
	# Include lock tests unless we're running on nfsv4
	local fstype=$(df -TP $testdir | awk 'NR==2  {print $2}')
	echo "$testdir: $fstype"
	if [[ $fstype != "nfs4" ]]; then
		tests="$tests -l"
	fi
	echo "tests: $tests"
	for test in $tests; do
		local cmd="sh ./runtests -N $cnt_NRUN $test -f $testdir"
		local rc=0

		log "$cmd"
		eval $cmd
		rc=$?
		[ $rc = 0 ] || error "connectathon failed: $rc"
	done

	cd $savePWD
	rm -rf $testdir
}

run_ior() {
	local type=${1:="ssf"}
	local dir=${2:-$DIR}
	local testdir=$dir/d0.ior.$type
	local nfs_srvmntpt=$3

	if [ "$NFSCLIENT" ]; then
		[[ -n $nfs_srvmntpt ]] ||
			{ error "NFSCLIENT mode, but nfs exported dir"\
				"is not set!" && return 1; }
	fi

	IOR=${IOR:-$(which IOR 2> /dev/null || true)}
	[ x$IOR = x ] && skip_env "IOR not found"

	# threads per client
	ior_THREADS=${ior_THREADS:-2}
	ior_iteration=${ior_iteration:-1}
	ior_blockSize=${ior_blockSize:-6}
	ior_blockUnit=${ior_blockUnit:-M}   # K, M, G
	ior_xferSize=${ior_xferSize:-1M}
	ior_type=${ior_type:-POSIX}
	ior_DURATION=${ior_DURATION:-30}        # minutes
	local multiplier=1
	case ${ior_blockUnit} in
		[G])
			multiplier=$((1024 * 1024 * 1024))
			;;
		[M])
			multiplier=$((1024 * 1024))
			;;
		[K])
			multiplier=1024
			;;
		*)      error "Incorrect block unit should be one of [KMG]"
			;;
	esac

	# calculate the space in bytes
	local space=$(df -B 1 -P $dir | tail -n 1 | awk '{ print $4 }')
	local total_threads=$((num_clients * ior_THREADS))
	echo "+ $ior_blockSize * $multiplier * $total_threads "
	if [ $((space / 2)) -le \
	     $((ior_blockSize * multiplier * total_threads)) ]; then
		ior_blockSize=$((space / 2 / multiplier / total_threads))
		[ $ior_blockSize -eq 0 ] &&
		skip_env "Need free space more than $((2 * total_threads)) \
			 ${ior_blockUnit}: have $((space / multiplier))"

		echo "(reduced blockSize to $ior_blockSize \
		     ${ior_blockUnit} bytes)"
	fi

	print_opts IOR ior_THREADS ior_DURATION MACHINEFILE

	test_mkdir -p $testdir

	# mpi_run uses mpiuser
	chmod 0777 $testdir
	[[ "$ior_stripe_params" && -z "$ior_STRIPEPARAMS" ]] &&
		ior_STRIPEPARAMS="$ior_stripe_params" &&
		echo "got deprecated ior_stripe_params,"\
			"use ior_STRIPEPARAMS instead"
	setstripe_getstripe $testdir $ior_STRIPEPARAMS

	#
	# -b N  blockSize --
	#       contiguous bytes to write per task (e.g.: 8, 4K, 2M, 1G)"
	# -o S  testFileName
	# -t N  transferSize -- size of transfer in bytes (e.g.: 8, 4K, 2M, 1G)"
	# -w    writeFile -- write file"
	# -r    readFile -- read existing file"
	# -W    checkWrite -- check read after write"
	# -C    reorderTasks -- changes task ordering to n+1 ordering for readback
	# -T    maxTimeDuration -- max time in minutes to run tests"
	# -k    keepFile -- keep testFile(s) on program exit

	local cmd
	if [ -n "$ior_custom_params" ]; then
		cmd="$IOR -o $testdir/iorData $ior_custom_params"
	else
		cmd="$IOR -a $ior_type -b ${ior_blockSize}${ior_blockUnit} \
		-o $testdir/iorData -t $ior_xferSize -v -C -w -r -W \
		-i $ior_iteration -T $ior_DURATION -k"
	fi

	[ $type = "fpp" ] && cmd="$cmd -F"

	echo "+ $cmd"
	# find out if we need to use srun by checking $SRUN_PARTITION
	if [ "$SRUN_PARTITION" ]; then
		$SRUN $SRUN_OPTIONS -D $testdir -w $clients -N $num_clients \
			-n $((num_clients * ior_THREADS)) -p $SRUN_PARTITION \
			-- $cmd
	else
		mpi_ior_custom_threads=${mpi_ior_custom_threads:-"$((num_clients * ior_THREADS))"}
		mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
			-np $mpi_ior_custom_threads $cmd
	fi

    local rc=$?
    if [ $rc != 0 ] ; then
        error "ior failed! $rc"
    fi
    rm -rf $testdir
}

run_mib() {
	MIB=${MIB:=$(which mib 2> /dev/null || true)}
	[ "$NFSCLIENT" ] && skip "skipped for NFSCLIENT mode"
	[ x$MIB = x ] && skip_env "MIB not found"

	# threads per client
	mib_THREADS=${mib_THREADS:-2}
	mib_xferSize=${mib_xferSize:-1m}
	mib_xferLimit=${mib_xferLimit:-5000}
	mib_timeLimit=${mib_timeLimit:-300}
	mib_STRIPEPARAMS=${mib_STRIPEPARAMS:-"-c -1"}

	print_opts MIB mib_THREADS mib_xferSize mib_xferLimit mib_timeLimit \
		MACHINEFILE

	local testdir=$DIR/d0.mib
	test_mkdir $testdir
	setstripe_getstripe $testdir $mib_STRIPEPARAMS

	# mpi_run uses mpiuser
	chmod 0777 $testdir

	#
	# -I    Show intermediate values in output
	# -H    Show headers in output
	# -L    Do not issue new system calls after this many seconds
	# -s    Use system calls of this size
	# -t    test dir
	# -l    Issue no more than this many system calls
	local cmd="$MIB -t $testdir -s $mib_xferSize -l $mib_xferLimit \
		-L $mib_timeLimit -HI -p mib.$(date +%Y%m%d%H%M%S)"

	echo "+ $cmd"
	# find out if we need to use srun by checking $SRUN_PARTITION
	if [ "$SRUN_PARTITION" ]; then
		$SRUN $SRUN_OPTIONS -D $testdir -w $clients -N $num_clients \
			-n $((num_clients * mib_THREADS)) -p $SRUN_PARTITION \
			-- $cmd
	else
		mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
			-np $((num_clients * mib_THREADS)) $cmd
	fi

    local rc=$?
    if [ $rc != 0 ] ; then
        error "mib failed! $rc"
    fi
    rm -rf $testdir
}

run_cascading_rw() {
	CASC_RW=${CASC_RW:-$(which cascading_rw 2> /dev/null || true)}
	[ x$CASC_RW = x ] && skip_env "cascading_rw not found"
	[ "$NFSCLIENT" ] && skip "skipped for NFSCLIENT mode"

	# threads per client
	casc_THREADS=${casc_THREADS:-2}
	casc_REP=${casc_REP:-300}

	# FIXME
	# Need space estimation here.

	print_opts CASC_RW clients casc_THREADS casc_REP MACHINEFILE

	local testdir=$DIR/d0.cascading_rw
	test_mkdir $testdir
	setstripe_getstripe $testdir $casc_STRIPEPARAMS

	# mpi_run uses mpiuser
	chmod 0777 $testdir

	# -g: debug mode
	# -n: repeat test # times

	local cmd="$CASC_RW -g -d $testdir -n $casc_REP"

	echo "+ $cmd"
	mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
		-np $((num_clients * $casc_THREADS)) $cmd

    local rc=$?
    if [ $rc != 0 ] ; then
        error "cascading_rw failed! $rc"
    fi
    rm -rf $testdir
}

run_write_append_truncate() {
	[ "$NFSCLIENT" ] && skip "skipped for NFSCLIENT mode"
	# location is lustre/tests dir
	if ! which write_append_truncate > /dev/null 2>&1 ; then
		skip_env "write_append_truncate not found"
	fi

	# threads per client
	write_THREADS=${write_THREADS:-8}
	write_REP=${write_REP:-10000}

	# FIXME
	# Need space estimation here.

	local testdir=$DIR/d0.write_append_truncate
	local file=$testdir/f0.wat

	print_opts clients write_REP write_THREADS MACHINEFILE

	test_mkdir $testdir
	# mpi_run uses mpiuser
	setstripe_getstripe $testdir $write_STRIPEPARAMS

	chmod 0777 $testdir

	local cmd="write_append_truncate -n $write_REP $file"

	echo "+ $cmd"
	mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
		-np $((num_clients * $write_THREADS)) $cmd

    local rc=$?
    if [ $rc != 0 ] ; then
        error "write_append_truncate failed! $rc"
        return $rc
    fi
    rm -rf $testdir
}

run_write_disjoint() {
	WRITE_DISJOINT=${WRITE_DISJOINT:-$(which write_disjoint 2> /dev/null ||
					   true)}
	[ x$WRITE_DISJOINT = x ] && skip_env "write_disjoint not found"
	[ "$NFSCLIENT" ] && skip "skipped for NFSCLIENT mode"

	# threads per client
	wdisjoint_THREADS=${wdisjoint_THREADS:-4}
	wdisjoint_REP=${wdisjoint_REP:-10000}
	chunk_size_limit=$1

	# FIXME
	# Need space estimation here.

	print_opts WRITE_DISJOINT clients wdisjoint_THREADS wdisjoint_REP \
		MACHINEFILE
	local testdir=$DIR/d0.write_disjoint
	test_mkdir $testdir
	setstripe_getstripe $testdir $wdisjoint_STRIPEPARAMS

	# mpi_run uses mpiuser
	chmod 0777 $testdir

	local cmd="$WRITE_DISJOINT -f $testdir/file -n $wdisjoint_REP -m \
			$chunk_size_limit"

	echo "+ $cmd"
	mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
		-np $((num_clients * $wdisjoint_THREADS)) $cmd

    local rc=$?
    if [ $rc != 0 ] ; then
        error "write_disjoint failed! $rc"
    fi
    rm -rf $testdir
}

run_parallel_grouplock() {
	PARALLEL_GROUPLOCK=${PARALLEL_GROUPLOCK:-$(which parallel_grouplock \
	    2> /dev/null || true)}

	[ x$PARALLEL_GROUPLOCK = x ] && skip "PARALLEL_GROUPLOCK not found"
	[ "$NFSCLIENT" ] && skip "skipped for NFSCLIENT mode"

	parallel_grouplock_MINTASKS=${parallel_grouplock_MINTASKS:-5}

	print_opts clients parallel_grouplock_MINTASKS MACHINEFILE

	local testdir=$DIR/d0.parallel_grouplock
	test_mkdir $testdir
	setstripe_getstripe $testdir $parallel_grouplock_STRIPEPARAMS

	# mpi_run uses mpiuser
	chmod 0777 $testdir

	local cmd
	local status=0
	local subtest
	for i in $(seq 12); do
		subtest="-t $i"
		local cmd="$PARALLEL_GROUPLOCK -g -v -d $testdir $subtest"
		echo "+ $cmd"

		mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
			-np $parallel_grouplock_MINTASKS $cmd
		local rc=$?
		if [ $rc != 0 ] ; then
			error_noexit "parallel_grouplock subtests $subtest " \
				     "failed! $rc"
		else
			echo "parallel_grouplock subtests $subtest PASS"
		fi
		let status=$((status + rc))
		# clear debug to collect one log per one test
		do_nodes $(comma_list $(nodes_list)) lctl clear
	done
	[ $status -eq 0 ] || error "parallel_grouplock status: $status"
	rm -rf $testdir
}

cleanup_statahead () {
    trap 0

    local clients=$1
    local mntpt_root=$2
    local num_mntpts=$3

    for i in $(seq 0 $num_mntpts);do
        zconf_umount_clients $clients ${mntpt_root}$i ||
            error_exit "Failed to umount lustre on ${mntpt_root}$i"
    done
}

run_statahead () {
	if [[ -n $NFSCLIENT ]]; then
		skip "Statahead testing is not supported on NFS clients."
	fi
	[ x$MDSRATE = x ] && skip_env "mdsrate not found"

	statahead_NUMMNTPTS=${statahead_NUMMNTPTS:-5}
	statahead_NUMFILES=${statahead_NUMFILES:-500000}

	print_opts MDSRATE clients statahead_NUMMNTPTS statahead_NUMFILES

	# create large dir

	# do not use default "d[0-9]*" dir name
	# to avoid of rm $statahead_NUMFILES (500k) files in t-f cleanup
	local dir=dstatahead
	local testdir=$DIR/$dir

	# cleanup only if dir exists
	# cleanup only $statahead_NUMFILES number of files
	# ignore the other files created by someone else
	[ -d $testdir ] &&
	mdsrate_cleanup $((num_clients * 32)) $MACHINEFILE \
		$statahead_NUMFILES $testdir 'f%%d' --ignore

	test_mkdir $testdir
	setstripe_getstripe $testdir $statahead_STRIPEPARAMS

    # mpi_run uses mpiuser
    chmod 0777 $testdir

    local num_files=$statahead_NUMFILES

    local IFree=$(inodes_available)
    if [ $IFree -lt $num_files ]; then
      num_files=$IFree
    fi

    cancel_lru_locks mdc

    local cmd1="${MDSRATE} ${MDSRATE_DEBUG} --mknod --dir $testdir"
    local cmd2="--nfiles $num_files --filefmt 'f%%d'"
    local cmd="$cmd1 $cmd2"
    echo "+ $cmd"

	mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
		-np $((num_clients * 32)) $cmd

    local rc=$?
    if [ $rc != 0 ] ; then
        error "mdsrate failed to create $rc"
        return $rc
    fi

    local num_mntpts=$statahead_NUMMNTPTS
    local mntpt_root=$TMP/mntpt/lustre
    local mntopts=$MNTOPTSTATAHEAD

    echo "Mounting $num_mntpts lustre clients starts on $clients"
    trap "cleanup_statahead $clients $mntpt_root $num_mntpts" EXIT ERR
    for i in $(seq 0 $num_mntpts); do
        zconf_mount_clients $clients ${mntpt_root}$i "$mntopts" ||
            error_exit "Failed to mount lustre on ${mntpt_root}$i on $clients"
    done

    do_rpc_nodes $clients cancel_lru_locks mdc

    do_rpc_nodes $clients do_ls $mntpt_root $num_mntpts $dir

    mdsrate_cleanup $((num_clients * 32)) $MACHINEFILE \
        $num_files $testdir 'f%%d' --ignore

    # use rm instead of rmdir because of
    # testdir could contain the files created by someone else,
    # or by previous run where is num_files prev > num_files current
    rm -rf $testdir
    cleanup_statahead $clients $mntpt_root $num_mntpts
}

cleanup_rr_alloc () {
	trap 0
	local clients="$1"
	local mntpt_root="$2"
	local rr_alloc_MNTPTS="$3"
	local mntpt_dir=$(dirname ${mntpt_root})

	for i in $(seq 0 $((rr_alloc_MNTPTS - 1))); do
		zconf_umount_clients $clients ${mntpt_root}$i ||
		error_exit "Failed to umount lustre on ${mntpt_root}$i"
	done
	do_nodes $clients "rm -rf $mntpt_dir"
}

run_rr_alloc() {
	remote_mds_nodsh && skip "remote MDS with nodsh"
	echo "===Test gives more reproduction percentage if number of "\
		"client and ost are more. Test with 44 or more clients "\
		"and 73 or more OSTs gives 100% reproduction rate=="

	RR_ALLOC=${RR_ALLOC:-$(which rr_alloc 2> /dev/null || true)}
	[ x$RR_ALLOC = x ] && skip_env "rr_alloc not found"
	declare -a diff_max_min_arr
	# foeo = file on each ost. calc = calculated.
	local ost_idx
	local foeo_calc
	local qos_prec_objs="${TMP}/qos_and_precreated_objects"
	local rr_alloc_NFILES=${rr_alloc_NFILES:-555}
	local rr_alloc_MNTPTS=${rr_alloc_MNTPTS:-11}
	local total_MNTPTS=$((rr_alloc_MNTPTS * num_clients))
	local mntpt_root="${TMP}/rr_alloc_mntpt/lustre"
	test_mkdir $DIR/$tdir
	setstripe_getstripe $DIR/$tdir $rr_alloc_STRIPEPARAMS

	chmod 0777 $DIR/$tdir

	trap "cleanup_rr_alloc $clients $mntpt_root $rr_alloc_MNTPTS" EXIT ERR
	for i in $(seq 0 $((rr_alloc_MNTPTS - 1))); do
		zconf_mount_clients $clients ${mntpt_root}$i $MOUNT_OPTS ||
		error_exit "Failed to mount lustre on ${mntpt_root}$i $clients"
	done

	local cmd="$RR_ALLOC $mntpt_root/$tdir/ash $rr_alloc_NFILES \
		$num_clients"

	# Save mdt values, set threshold to 100% i.e always Round Robin,
	# restore the saved values again after creating files...
	save_lustre_params mds1 \
		"lov.$FSNAME-MDT0000*.qos_threshold_rr" > $qos_prec_objs
	save_lustre_params mds1 \
		"osp.$FSNAME-OST*-osc-MDT0000.create_count" >> $qos_prec_objs

	local old_create_count=$(grep -e "create_count" $qos_prec_objs |
		cut -d'=' -f 2 | sort -nr | head -n1)

	# Make sure that every osp has enough precreated objects for the file
	# creation app

	# create_count is always set to the power of 2 only, so if the files
	# per OST are not multiple of that then it will be set to nearest
	# lower power of 2. So set 'create_count' to the upper power of 2.

	foeo_calc=$((rr_alloc_NFILES * total_MNTPTS / OSTCOUNT))
	local create_count=$((2 * foeo_calc))
	do_facet mds1 "$LCTL set_param -n \
		lov.$FSNAME-MDT0000*.qos_threshold_rr 100 \
		osp.$FSNAME-OST*-osc-MDT0000.create_count $create_count" ||
		error "failed while setting qos_threshold_rr & creat_count"

	# Create few temporary files in order to increase the precreated objects
	# to a desired value, before starting 'rr_alloc' app. Due to default
	# value 32 of precreation count (OST_MIN_PRECREATE=32), precreated
	# objects available are 32 initially, these gets exhausted very soon,
	# which causes skip of some osps when very large number of files
	# is created per OSTs.
	createmany -o $DIR/$tdir/foo- $(((old_create_count + 1) * OSTCOUNT)) \
		> /dev/null
	rm -f /$DIR/$tdir/foo*

	# Check for enough precreated objects... We should not
	# fail here because code(osp_precreate.c) also takes care of it.
	# So we have good chances of passing test even if this check fails.
	local mdt_idx=0
	for ost_idx in $(seq 0 $((OSTCOUNT - 1))); do
		[[ $(precreated_ost_obj_count $mdt_idx $ost_idx) -ge \
			$foeo_calc ]] || echo "Warning: test may fail because" \
			"of lack of precreated objects on OST${ost_idx}"
	done

	if [[ $total_MNTPTS -ne 0 ]]; then
		# Now start the actual file creation app.
		mpi_run "-np $total_MNTPTS" $cmd || return
	else
		error "No mount point"
	fi

	restore_lustre_params < $qos_prec_objs
	rm -f $qos_prec_objs

	diff_max_min_arr=($($LFS getstripe -r $DIR/$tdir/ |
		grep "lmm_stripe_offset:" | awk '{print $2}' | sort -n |
		uniq -c | awk 'NR==1 {min=max=$1} \
		{ $1<min ? min=$1 : min; $1>max ? max=$1 : max} \
		END {print max-min, max, min}'))

	rm -rf $DIR/$tdir

	# In-case of fairly large number of file creation using RR (round-robin)
	# there can be two cases in which deviation will occur than the regular
	# RR algo behaviour-
	# 1- When rr_alloc does not start right with 'lqr_start_count' reseeded,
	# 2- When rr_alloc does not finish with 'lqr_start_count == 0'.
	# So the difference of files b/w any 2 OST should not be more than 2.
	[[ ${diff_max_min_arr[0]} -le 2 ]] ||
		error "Uneven distribution detected: difference between" \
		"maximum files per OST (${diff_max_min_arr[1]}) and" \
		"minimum files per OST (${diff_max_min_arr[2]}) must not be" \
		"greater than 2"
}

run_fs_test() {
	# fs_test.x is the default name for exe
	FS_TEST=${FS_TEST:=$(which fs_test.x 2> /dev/null || true)}

	local clients=${CLIENTS:-$(hostname)}
	local testdir=$DIR/d0.fs_test
	local file=${testdir}/fs_test
	fs_test_threads=${fs_test_threads:-2}
	fs_test_type=${fs_test_type:-1}
	fs_test_nobj=${fs_test_nobj:-10}
	fs_test_check=${fs_test_check:-3}
	fs_test_strided=${fs_test_strided:-1}
	fs_test_touch=${fs_test_touch:-3}
	fs_test_supersize=${fs_test_supersize:-1}
	fs_test_op=${fs_test_op:-write}
	fs_test_barriers=${fs_test_barriers:-bopen,bwrite,bclose}
	fs_test_io=${fs_test_io:-mpi}
	fs_test_objsize=${fs_test_objsize:-100}
	fs_test_objunit=${fs_test_objunit:-1048576} # 1 mb
	fs_test_ndirs=${fs_test_ndirs:-80000}

	[ x$FS_TEST = x ] && skip "FS_TEST not found"

	# Space estimation  in bytes
	local space=$(df -B 1 -P $dir | tail -n 1 | awk '{ print $4 }')
	local total_threads=$((num_clients * fs_test_threads))
	echo "+ $fs_test_objsize * $fs_test_objunit * $total_threads "
	if [ $((space / 2)) -le \
		$((fs_test_objsize * fs_test_objunit * total_threads)) ]; then
			fs_test_objsize=$((space / 2 / fs_test_objunit / \
				total_threads))
			[ $fs_test_objsize -eq 0 ] &&
			skip_env "Need free space more than \
				$((2 * total_threads * fs_test_objunit)) \
				: have $((space / fs_test_objunit))"

			echo "(reduced objsize to \
				$((fs_test_objsize * fs_test_objunit)) bytes)"
	fi

	print_opts FS_TEST clients fs_test_threads fs_test_objsize MACHINEFILE

	test_mkdir $testdir
	setstripe_getstripe $testdir $fs_test_STRIPEPARAMS

	# mpi_run uses mpiuser
	chmod 0777 $testdir

	# --nodb          Turn off the database code at runtime
	# -g --target     The path to the data file
	# -t --type       Whether to do N-N (1) or N-1 (2)
	# -n --nobj       The number of objects written/read by each proc
	# -z --size       The size of each object
	# -d ---num_nn_dirs Number of subdirectories for files
	# -C --check      Check every byte using argument 3.
	# --collective    Whether to use collective I/O (for N-1, mpi-io only)
	# -s --strided    Whether to use a strided pattern (for N-1 only)
	# -T --touch      Touch every byte using argument 3
	# -o --op         Whether to read only (read) or write only (write)
	# -b --barriers   When to barrier.
	# -i --io         Use POSIX, MPI, or PLFS IO routines (mpi|posix|plfs)
	# -S --supersize  Specify how many objects per superblock

	local cmd="$FS_TEST -nodb -g $file -t $fs_test_type -n $fs_test_nobj \
		-z $((fs_test_objsize * fs_test_objunit)) -d $fs_test_ndirs \
		-C $fs_test_check -collective -s $fs_test_strided \
		-T $fs_test_touch -o $fs_test_op -b $fs_test_barriers \
		-i $fs_test_io -S $fs_test_supersize"

	echo "+ $cmd"
	mpi_run "-np $((num_clients * fs_test_threads))" $cmd

	local rc=$?
	if [ $rc != 0 ] ; then
		error "fs_test failed! $rc"
	fi

	rm -rf $testdir
}

ior_mdtest_parallel() {
	local rc1=0
	local rc2=0
	local type=$1

	run_ior $type &
	local pids=$!

	run_mdtest $type || rc2=$?
	[[ $rc2 -ne 0 ]] && echo "mdtest failed with error $rc2"

	wait $pids || rc1=$?
	[[ $rc1 -ne 0 ]] && echo "ior failed with error $rc1"

	[[ $rc1 -ne 0 || $rc2 -ne 0 ]] && return 1
	return 0
}

run_fio() {
	FIO=${FIO:=$(which fio 2> /dev/null || true)}

	local clients=${CLIENTS:-$(hostname)}
	local fio_jobNum=${fio_jobNum:-4}
	local fio_jobFile=${fio_jobFile:-$TMP/fiojobfile.$(date +%s)}
	local fio_bs=${fio_bs:-1}
	local testdir=$DIR/d0.fio
	local file=${testdir}/fio
	local runtime=60
	local propagate=false

	[ "$SLOW" = "no" ] || runtime=600

	[ x$FIO = x ] && skip_env "FIO not found"

	test_mkdir $testdir
	setstripe_getstripe $testdir $fio_STRIPEPARAMS

	# use fio job file if exists,
	# create a simple one if missing
	if ! [ -f $fio_jobFile ]; then
		cat >> $fio_jobFile <<EOF
[global]
rw=randwrite
size=128m
time_based=1
runtime=$runtime
filename=${file}_\$(hostname)
EOF
		# bs size increased by $i for each job
		for ((i=1; i<=fio_jobNum; i++)); do
			cat >> $fio_jobFile <<EOF

[job$i]
bs=$(( fio_bs * i ))m
EOF
		done
		# job file is created, should be propagated to all clients
		propagate=true
	fi

	# propagate the job file if not all clients have it yet or
	# if the job file was created during the test run
	if ! do_nodesv $clients " [ -f $fio_jobFile ] " ||
	   $propagate; then
		local cfg=$(cat $fio_jobFile)
		do_nodes $clients "echo \\\"$cfg\\\" > ${fio_jobFile}" ||
			error "job file $fio_jobFile is not propagated"
		do_nodesv $clients "cat ${fio_jobFile}"
	fi

	cmd="$FIO $fio_jobFile"
	echo "+ $cmd"

	log "clients: $clients $cmd"

	local rc=0
	do_nodesv $clients "$cmd "
	rc=$?

	[ $rc = 0 ] || error "fio failed: $rc"
	rm -rf $testdir
}

run_xdd() {
	XDD=${XDD:=$(which xdd 2> /dev/null || true)}

	local clients=${CLIENTS:-$(hostname)}
	local testdir=$DIR/d0.xdd
	xdd_queuedepth=${xdd_queuedepth:-4}
	xdd_blocksize=${xdd_blocksize:-512}
	xdd_reqsize=${xdd_reqsize:-128}
	xdd_mbytes=${xdd_mbytes:-100}
	xdd_passes=${xdd_passes:-40}
	xdd_rwratio=${xdd_rwratio:-0}
	xdd_ntargets=${xdd_ntargets:-6}
	local xdd_custom_params=${xdd_custom_params:-"-dio -stoponerror \
		-maxpri -minall -noproclock -nomemlock"}

	[ x$XDD = x ] && skip "XDD not found"

	print_opts XDD clients xdd_queuedepth xdd_blocksize xdd_reqsize \
		xdd_mbytes xdd_passes xdd_rwratio

	test_mkdir $testdir
	setstripe_getstripe $testdir $xdd_STRIPEPARAMS

	local files=""
	# Target files creates based on the given number of targets
	for (( i=0; i < $xdd_ntargets; i++ ))
	do
		files+="${testdir}/xdd"$i" "
	done

	# -targets      specifies the devices or files to perform operation
	# -reqsize      number of 'blocks' per operation
	# -mbytes       number of 1024*1024-byte blocks to transfer
	# -blocksize    size of a single 'block'
	# -passes       number of times to read mbytes
	# -queuedepth   number of commands to queue on the target
	# -rwratio      percentage of read to write operations
	# -verbose      will print out statistics on each pass

	local cmd="$XDD -targets $xdd_ntargets $files -reqsize $xdd_reqsize \
		-mbytes $xdd_mbytes -blocksize $xdd_blocksize \
		-passes $xdd_passes -queuedepth $xdd_queuedepth \
		-rwratio $xdd_rwratio -verbose $xdd_custom_params"
	echo "+ $cmd"

	local rc=0
	do_nodesv $clients "$cmd "
	rc=$?

	[ $rc = 0 ] || error "xdd failed: $rc"

	rm -rf $testdir
}