#!/bin/bash

set -e

ONLY=${ONLY:-"$*"}
ALWAYS_EXCEPT="$SCRUB_PERFORMANCE_EXCEPT"
[ "$SLOW" = "no" ] && EXCEPT_SLOW=""
# UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!

LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
. $LUSTRE/tests/test-framework.sh
init_test_env $@
. ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
init_logging

[ "$SLOW" = "no" ] &&
	skip "skip scrub performance test under non-SLOW mode" && exit 0

[ "$mds1_FSTYPE" != ldiskfs ] &&
	skip "ldiskfs only test"
[[ "$MDS1_VERSION" -lt $(version_code 2.2.90) ]] &&
	skip "Need MDS version at least 2.2.90"
require_dsh_mds || exit 0

NTHREADS=${NTHREADS:-0}
UNIT=${UNIT:-1048576}
BACKUP=${BACKUP:-0}
MINCOUNT=${MINCOUNT:-8192}
MAXCOUNT=${MAXCOUNT:-32768}
FACTOR=${FACTOR:-2}

RCMD="do_facet ${SINGLEMDS}"
RLCTL="${RCMD} ${LCTL}"
MDT_DEV="${FSNAME}-MDT0000"
MDT_DEVNAME=$(mdsdevname ${SINGLEMDS//mds/})
SHOW_SCRUB="${RLCTL} get_param -n osd-ldiskfs.${MDT_DEV}.oi_scrub"
remote_mds && ECHOCMD=${RCMD} || ECHOCMD="eval"

if [ ${NTHREADS} -eq 0 ]; then
	CPUCORE=$(${RCMD} cat /proc/cpuinfo | grep "processor.*:" | wc -l)
	NTHREADS=$((CPUCORE * 2))
fi

cleanupall

if ! combined_mgs_mds ; then
	do_rpc_nodes $(facet_active_host mgs) load_modules_local
	add mgs $(mkfs_opts mgs $(mgsdevname)) --backfstype ldiskfs \
		--reformat $(mgsdevname) $(mgsvdevname) ${quiet:+>/dev/null} ||
		exit 1

	start mgs $(mgsdevname) $MGS_MOUNT_OPTS || error "Fail to start MGS!"
fi

do_rpc_nodes $(facet_active_host $SINGLEMDS) load_modules_local
reformat_external_journal ${SINGLEMDS}
add ${SINGLEMDS} $(mkfs_opts ${SINGLEMDS} ${MDT_DEVNAME}) --backfstype ldiskfs \
	--reformat ${MDT_DEVNAME} $(mdsvdevname 1) ${quiet:+>/dev/null} ||
	exit 2

scrub_attach() {
	${ECHOCMD} "${LCTL} <<-EOF
		attach echo_client scrub-MDT0000 scrub-MDT0000_UUID
		setup ${MDT_DEV} mdd
	EOF"
}

scrub_detach() {
	${ECHOCMD} "${LCTL} <<-EOF
		device scrub-MDT0000
		cleanup
		detach
	EOF"
}

scrub_create() {
	local echodev=$(${RLCTL} dl | grep echo_client|awk '{print $1}')
	local j

	${ECHOCMD} "${LCTL} <<-EOF
		cfg_device ${echodev}
		test_mkdir ${tdir}
	EOF"

	for ((j=1; j<${threads}; j++)); do
		${ECHOCMD} "${LCTL} <<-EOF
			cfg_device ${echodev}
			test_mkdir ${tdir}${j}
		EOF"
	done

	${ECHOCMD} "${LCTL} <<-EOF
		cfg_device ${echodev}
		--threads ${threads} 0 ${echodev} test_create \
		-d ${tdir} -D ${threads} -b ${lbase} -c 0 -n ${usize}
	EOF"
}

scrub_cleanup() {
	cleanupall
	do_rpc_nodes $(facet_active_host $SINGLEMDS) unload_modules
	if ! combined_mgs_mds ; then
		do_rpc_nodes $(facet_active_host mgs) unload_modules
	fi
	REFORMAT="yes" cleanup_and_setup_lustre
}

scrub_create_nfiles() {
	local total=$1
	local lbase=$2
	local threads=$3
	local ldir="/test-${lbase}"
	local cycle=0
	local count=${UNIT}

	while true; do
		[ ${count} -eq 0 -o  ${count} -gt ${total} ] && count=${total}
		local usize=$((count / NTHREADS))
		[ ${usize} -eq 0 ] && break
		local tdir=${ldir}-${cycle}-

		echo "[cycle: ${cycle}] [threads: ${threads}]"\
		     "[files: ${count}] [basedir: ${tdir}]"
		start ${SINGLEMDS} $MDT_DEVNAME $MDS_MOUNT_OPTS ||
			error "Fail to start MDS!"
		scrub_attach
		scrub_create
		scrub_detach
		stop ${SINGLEMDS} || error "Fail to stop MDS!"

		total=$((total - usize * NTHREADS))
		[ ${total} -eq 0 ] && break
		lbase=$((lbase + usize))
		cycle=$((cycle + 1))
	done
}

build_test_filter

test_0() {
	local BASECOUNT=0
	local i

	for ((i=$MINCOUNT; i<=$MAXCOUNT; i=$((i * FACTOR)))); do
		local nfiles=$((i - BASECOUNT))
		local stime=$(date +%s)

		echo "+++ start to create for ${i} files set at: $(date) +++"
		scrub_create_nfiles ${nfiles} ${BASECOUNT} ${NTHREADS} ||
			error "Fail to create files!"
		echo "+++ end to create for ${i} files set at: $(date) +++"
		local etime=$(date +%s)
		local delta=$((etime - stime))
		[ $delta -gt 0 ] || delta=1
		echo "create ${nfiles} files used ${delta} seconds"
		echo "create speed is $((nfiles / delta))/sec"

		BASECOUNT=${i}
		if [ ${BACKUP} -ne 0 ]; then
			stime=$(date +%s)
			echo "backup/restore ${i} files start at: $(date)"
			mds_backup_restore $SINGLEMDS ||
				error "Fail to backup/restore!"
			echo "backup/restore ${i} files end at: $(date)"
			etime=$(date +%s)
			delta=$((etime - stime))
			[ $delta -gt 0 ] || delta=1
			echo "backup/restore ${i} files used ${delta} seconds"
			echo "backup/restore speed is $((i / delta))/sec"
		else
			mds_remove_ois $SINGLEMDS ||
				error "Fail to remove/recreate!"
		fi

		echo "--- start to rebuild OI for $i files set at: $(date) ---"
		start ${SINGLEMDS} $MDT_DEVNAME $MDS_MOUNT_OPTS > /dev/null ||
			error "Fail to start MDS!"

		while true; do
			local STATUS=$($SHOW_SCRUB |
					awk '/^status/ { print $2 }')
			[ "$STATUS" == "completed" ] && break
			sleep 3 # check status every 3 seconds
		done

		echo "--- end to rebuild OI for ${i} files set at: $(date) ---"
		local RTIME=$($SHOW_SCRUB | awk '/^run_time/ { print $2 }')
		echo "rebuild OI for ${i} files used ${RTIME} seconds"
		local SPEED=$($SHOW_SCRUB | awk '/^average_speed/ { print $2 }')
		echo "rebuild speed is ${SPEED}/sec"
		stop ${SINGLEMDS} > /dev/null || error "Fail to stop MDS!"
	done
}
run_test 0 "OI scrub performance test"

# cleanup the system at last
scrub_cleanup
complete $SECONDS
check_and_cleanup_lustre
exit_status


#!/bin/bash

# Requires the pre-configured samba machine
# RPMS required are :
# server:
#      samba
#      samba-common
#      cifs-utils
# clients:
#      samba-client
#      samba-common
#      cifs-utils

#set -vx

smb_status() {
	local smbsrv=${1}
	local rc=0

	do_node $smbsrv "service smb status" || rc=$?
	return $rc
}

configure_smb() {
	local smbsrv=${1}
	local smbshare=${2}
	local smbuser=${3}
	local smbpasswd=${4}
	local path=${5}
	local smbconftmp=${6}
	local smbconf
	local path

	smbconf=$(do_node $smbsrv \
		smbd -b | grep CONFIGFILE | sed -re 's/\s+CONFIGFILE: //g')
	grep -q $smbshare $smbconf && return

	do_node $smbsrv "cp $smbconf $smbconftmp"
	do_node $smbsrv "cat <<EOF >> $smbconf
	[$smbshare]
	path = $path
	browseable = yes
	writable = yes
	guest ok = yes
	write list = $smbuser
EOF"
	# The samba daemons are constantly (once every 60 seconds) checking the
	# smb.conf file, so it is good practice to keep this file small.
	local smbsmall=$(do_nodes $smbsrv "mktemp -t smb.conf.small.XXX")
	do_node $smbsrv "testparm -s >$smbsmall"
	do_node $smbsrv "testparm -s $smbsmall >$smbconf"

	do_node $smbsrv "echo $smbpasswd | tee - | smbpasswd -a $smbuser -s"
}

restore_config_smb() {
	local smbsrv=${1}
	local smbconftmp=${2}
	local smbconf

	smbconf=$(do_node $smbsrv \
		smbd -b | grep CONFIGFILE | sed -re 's/\s+CONFIGFILE: //g')

	echo -e "\nRestoring smb config from $smbconftmp ..."
	do_node $smbsrv "cp $smbconftmp $smbconf"
}

setup_cifs() {
	local smbsrv=${1}
	local smbshare=${2}
	local smbclimntpt=${3}
	local smbuser=${4}
	local smbpasswd=${5}
	local smbclients=${6}

	do_node $smbsrv "service smb restart" || return 1
	local parameter_path=$(do_node $smbsrv \
		"testparm -s --section-name $smbshare --parameter-name path 2> /dev/null")
	[[ -z $parameter_path ]] && return 1
	do_nodesv $smbsrv "chmod a+xrw $parameter_path"
	do_nodesv $smbsrv "ls -ald $parameter_path"

	local cmd="mount -t cifs //$smbsrv/$smbshare $smbclimntpt -o user=$smbuser,pass=$smbpasswd"
	echo -e "\nMounting CIFS clients $smbclients : $cmd"
	do_nodesv $smbclients "$cmd" || return 1
	do_nodesv $smbclients mount | grep $smbclimntpt
}

cleanup_cifs() {
	local smbsrv=${1}
	local smbclimntpt=${2}
	local smbclients=${3}

	echo -e "\nUnmounting CIFS clients..."
	do_nodes $smbclients "umount -f $smbclimntpt" || return 1
	do_node $smbsrv "service smb stop"
}


#!/bin/bash
#set -x
EXPORT_OPTS=${EXPORT_OPTS:-"rw,async,no_root_squash"}

setup_nfs() {
	local NFS_VER=${1}
	local MNTPNT=${2}
	local LUSTRE_CLIENT=${3}
	local NFS_CLIENTS=${4}
	local nfs_climntpt=${5:-$MNTPNT}

    local export_opts_v=$EXPORT_OPTS

    echo "Exporting Lustre filesystem..."

    if [ "$NFS_VER" = "4" ]; then
        export_opts_v="$EXPORT_OPTS,fsid=0"
        do_nodes $LUSTRE_CLIENT "mkdir -p /var/lib/nfs/v4recovery"
    fi

    do_nodes $LUSTRE_CLIENT,$NFS_CLIENTS "grep -q rpc_pipefs' ' /proc/mounts ||\
        { mkdir -p /var/lib/nfs/rpc_pipefs && \
        mount -t rpc_pipefs sunrpc /var/lib/nfs/rpc_pipefs; }" || return 1
    sleep 5

	do_nodes $LUSTRE_CLIENT "echo $MNTPNT *\($export_opts_v\) \
				 >> /etc/exports" || return 1

	# restart nfs server according to distro
	do_nodes $LUSTRE_CLIENT "{ [[ -e /etc/SuSE-release ]] &&
				 service nfsserver restart; } ||
				 service nfs restart ||
				 service nfs-server restart" || return 1

	do_nodes $NFS_CLIENTS "chkconfig --list rpcidmapd 2>/dev/null |
			       grep -q rpcidmapd && service rpcidmapd restart ||
			       true"

	echo -e "\nMounting NFS clients (version $NFS_VER)..."

	do_nodes $NFS_CLIENTS "mkdir -p $nfs_climntpt" || return 1
	if [ "$NFS_VER" = "4" ]; then
		do_nodes $NFS_CLIENTS \
			"mount -t nfs$NFS_VER -o async \
			$LUSTRE_CLIENT:/ $nfs_climntpt" || return 1
	else
		do_nodes $NFS_CLIENTS \
			"mount -t nfs -o nfsvers=$NFS_VER,async \
			$LUSTRE_CLIENT:$MNTPNT $nfs_climntpt" || return 1
	fi
	return 0
}

cleanup_nfs() {
	local MNTPNT=${1}
	local LUSTRE_CLIENT=${2}
	local NFS_CLIENTS=${3}

	echo -e "\nUnmounting NFS clients..."
	do_nodes $NFS_CLIENTS "umount -f $MNTPNT" || true

	echo -e "\nUnexporting Lustre filesystem..."
	do_nodes $NFS_CLIENTS "chkconfig --list rpcidmapd 2>/dev/null |
			       grep -q rpcidmapd && service rpcidmapd stop ||
			       true"

	do_nodes $LUSTRE_CLIENT "{ [[ -e /etc/SuSE-release ]] &&
				 service nfsserver stop; } ||
				 service nfs stop ||
				 service nfs-server stop" || return 1

	do_nodes $LUSTRE_CLIENT "sed -i '/${MNTPNT##*/}/d' /etc/exports" || return 1

	do_nodes $LUSTRE_CLIENT "exportfs -v"
}


#!/bin/bash
# -*- mode: Bash; tab-width: 4; indent-tabs-mode: t; -*-
# vim:shiftwidth=4:softtabstop=4:tabstop=4:

#
# setup_kerberos.sh - setup the Kerberos environment on Lustre cluster
#
# Notes:
#  * Only one KDC involved, no slave KDC.
#  * Only one Kerberos realm involved, no multiple Kerberos realms.
#
###############################################################################

# usage
my_usage() {
    cat <<EOF
Usage: $(basename $0) <KDC_distro> <KDC_node> <MGS_node> <MDS_node>[:MDS_node:...]
                      <OSS_node>[:OSS_node:...] <CLIENT_node>[:CLIENT_node:...]

    This script is used to setup the Kerberos environment on Lustre cluster.

    KDC_distro      distribution on the KDC node (rhel5 or sles10)
    KDC_node        KDC node name
    MGS_node        Lustre MGS node name
    MDS_node        Lustre MDS node name
    OSS_node        Lustre OSS node name
    CLIENT_node     Lustre client node name

    e.g.: $(basename $0) rhel5 scsi2 scsi2 sata2 sata3 client5
    e.g.: $(basename $0) sles10 scsi2 scsi2 scsi2 sata3:sata5 client2:client3
    e.g.: $(basename $0) rhel5 scsi2 scsi2 scsi2 scsi2 scsi2

    Notes:
    1) The script will destroy all the old Kerberos settings by default. If you
    want to reserve the original krb5.conf and KDC configuration, please set
    "RESET_KDC=false".

    2) The script will create principals for some runas users and add them into
    the Kerberos database by default. The UIDs of the runas users specified in
    "LOCAL_UIDS" variable need exist on KDC, MDS and Client nodes. If you do not
    need runas users, please set "CFG_RUNAS=false".

EOF
}

# ************************ Parameters and Variables ************************ #
MY_KDC_DISTRO=$1
MY_KDCNODE=$2
MY_MGSNODE=$3
MY_MDSNODES=$4
MY_OSSNODES=$5
MY_CLIENTNODES=$6

# translate to lower case letters
MY_KDC_DISTRO=$(echo $MY_KDC_DISTRO | tr '[A-Z]' '[a-z]')

if [ -z "$MY_KDC_DISTRO" -o -z "$MY_KDCNODE" -o -z "$MY_MDSNODES" -o \
    -z "$MY_OSSNODES" -o -z "$MY_CLIENTNODES" -o -z "$MY_MGSNODE" ]; then
    my_usage
    exit 1
fi

LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
. $LUSTRE/tests/test-framework.sh
init_test_env
. ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}

SCP=${SCP:-"scp -q"}
ACCEPTOR_PORT=${ACCEPTOR_PORT:-988}

# check and configure runas users
CFG_RUNAS=${CFG_RUNAS:-true}
# uids for local users
LOCAL_UIDS=${LOCAL_UIDS:-"500 501"}

# remove the original Kerberos and KDC settings
RESET_KDC=${RESET_KDC:-true}

# generate unique keytab for each client node
SPLIT_KEYTAB=${SPLIT_KEYTAB:-true}

# encryption types for generating keytab
MDS_ENCTYPE=${MDS_ENCTYPE:-"aes128-cts"}
MGS_ENCTYPE=${MGS_ENCTYPE:-"$MDS_ENCTYPE"}
OSS_ENCTYPE=${OSS_ENCTYPE:-"aes128-cts"}
CLIENT_ENCTYPE=${CLIENT_ENCTYPE:-"aes128-cts"}

# configuration file for Kerberos
KRB5_CONF=${KRB5_CONF:-"/etc/krb5.conf"}
KRB5_KEYTAB=${KRB5_KEYTAB:-"/etc/krb5.keytab"}
KRB5_TICKET_LIFETIME=${KRB5_TICKET_LIFETIME:-"24h"}

# configuration files for libgssapi and keyutils
GSSAPI_MECH_CONF=${GSSAPI_MECH_CONF:-"/etc/gssapi_mech.conf"}
REQUEST_KEY_CONF=${REQUEST_KEY_CONF:-"/etc/request-key.conf"}

# krb5 realm & domain
KRB5_REALM=${KRB5_REALM:-"CO.CFS"}
KRB5_DOMAIN=$(echo $KRB5_REALM | tr '[A-Z]' '[a-z]')

MY_MDSNODES=${MY_MDSNODES//:/ }
MY_OSSNODES=${MY_OSSNODES//:/ }
MY_CLIENTNODES=${MY_CLIENTNODES//:/ }

# set vars according to the KDC distribution
KRB5PKG_SVR="krb5-server"
KRB5PKG_DEV="krb5-devel"
case $MY_KDC_DISTRO in
    rhel5)
        KRB5PKG_CLI="krb5-workstation"
        KRB5PKG_LIB="krb5-libs"
        KDC_CONF_DIR="/var/kerberos/krb5kdc"
        ;;
    sles10)
        KRB5PKG_CLI="krb5-client"
        KRB5PKG_LIB="krb5"
        KDC_CONF_DIR="/var/lib/kerberos/krb5kdc"
        ;;
    *)
        echo "Unsupported KDC distro: $MY_KDC_DISTRO!"
        exit 1
esac
KDC_CONF="$KDC_CONF_DIR/kdc.conf"
KDC_ACL="$KDC_CONF_DIR/kadm5.acl"

# ******************************** Functions ******************************** #
is_part_of() {
    local name="$1"
    shift
    local list="$@"

    if [ -z "$name" -o -z "$list" ]; then
        false
        return
    fi

    if [[ " $list " == *\ $name\ * ]]; then
        true
    else
        false
    fi

    return
}

my_do_node() {
    local node=$1
    shift
    local nodename=${node%.$KRB5_DOMAIN}
    do_node $node "PATH=\$PATH:/usr/kerberos/sbin:/usr/kerberos/bin:\
/usr/lib/mit/sbin:/usr/lib/mit/bin $@" | sed "s/^${nodename}: //"
    return ${PIPESTATUS[0]}
}

do_node_mute() {
    local output
    output=$(my_do_node "$@" 2>&1)
    return ${PIPESTATUS[0]}
}

do_kdc() {
    my_do_node $MY_KDCNODE "$@"
    return ${PIPESTATUS[0]}
}

do_kdc_mute() {
    do_node_mute $MY_KDCNODE "$@"
    return ${PIPESTATUS[0]}
}

#
# convert a space-delimited node name list to a canonical name list
#
get_fqdn() {
    local nodename_list="$@"
    local fqdn_list=""
    local name
    local fqdn
    local rc

    for name in $nodename_list; do
        fqdn=$(do_kdc "gethostip -n $name 2>&1")
        rc=${PIPESTATUS[0]}
        if [ $rc -ne 0 ]; then
            echo "Can not get the FQDN of node $name: $fqdn"
            return $rc
        fi
        [ -z "$fqdn_list" ] && fqdn_list="$fqdn" \
        || fqdn_list="$fqdn_list $fqdn"

    done

    echo "$fqdn_list"
    return 0
}

#
# convert MDS/OSS nodes to their canonical name, it required by
# kerberos. we also convert kdc and client too in order to make
# node name comparison easier
#
normalize_names() {
    local rc

    # KDC
    MY_KDCNODE=$(get_fqdn $MY_KDCNODE)
    rc=${PIPESTATUS[0]}
    if [ $rc -ne 0 ]; then
        echo $MY_KDCNODE
        return $rc
    fi

    # MGS node
    MY_MGSNODE=$(get_fqdn $MY_MGSNODE)
    rc=${PIPESTATUS[0]}
    if [ $rc -ne 0 ]; then
        echo $MY_MGSNODE
        return $rc
    fi

    # MDS nodes
    MY_MDSNODES=$(get_fqdn $MY_MDSNODES)
    rc=${PIPESTATUS[0]}
    if [ $rc -ne 0 ]; then
        echo $MY_MDSNODES
        return $rc
    fi

    # OSS nodes
    MY_OSSNODES=$(get_fqdn $MY_OSSNODES)
    rc=${PIPESTATUS[0]}
    if [ $rc -ne 0 ]; then
        echo $MY_OSSNODES
        return $rc
    fi

    # client nodes
    MY_CLIENTNODES=$(get_fqdn $MY_CLIENTNODES)
    rc=${PIPESTATUS[0]}
    if [ $rc -ne 0 ]; then
        echo $MY_CLIENTNODES
        return $rc
    fi

    return 0
}

#
# verify remote shell works on all nodes
#
check_rsh() {
    local checked=""
    local node

    echo "+++ Checking remote shell"

    for node in $MY_KDCNODE $MY_MGSNODE $MY_OSSNODES $MY_MDSNODES $MY_CLIENTNODES
    do
        is_part_of $node $checked && continue

        echo -n "Checking remote shell on $node..."
        do_node_mute $node true || return ${PIPESTATUS[0]}
        echo "OK!"

        checked="$checked $node"
    done
}

#
# verify the entropy (random numbers) on the KDC node, which is
# used by kdb5_util to create Kerberos database
#
check_entropy() {
    local limit=170
    local avail

    echo "+++ Checking the entropy on the KDC"

    echo -n "Checking $MY_KDCNODE..."
    avail=$(do_kdc "sysctl -n kernel.random.entropy_avail")
    local rc=${PIPESTATUS[0]}
    if [ $rc -eq 0 ]; then
        if [ $avail -lt $limit ]; then
            echo -e "\nWarning: The entropy on the KDC node is only $avail, \
which is not enough for kdb5_util to create Kerberos database! \
Let's use /dev/urandom!"
            do_kdc "rm -f /dev/random.bak && mv /dev/random{,.bak} && \
mknod /dev/random c 1 9"
            return ${PIPESTATUS[0]}
        fi
    else
        echo "Can not get the entropy on the KDC node!"
        return $rc
    fi
    echo "OK!"
}

#
# verify runas users and groups
#
check_users() {
    local checked=""
    local node
    local id
    local user

    echo "+++ Checking users and groups"

    for node in $MY_KDCNODE $MY_MGSNODE $MY_MDSNODES $MY_CLIENTNODES; do
        is_part_of $node $checked && continue

        for id in $LOCAL_UIDS; do
            echo -n "Checking uid/gid $id/$id on $node..."
            user=$(my_do_node $node getent passwd | grep :$id:$id: | cut -d: -f1)
            if [ -z "$user" ]; then
				echo -e "\nPlease set LOCAL_UIDS to some users \
which exist on KDC, MDS and client or add user/group $id/$id on these nodes."
                return 1
            fi
            echo "OK!"
        done
        checked="$checked $node"
    done
}

cfg_mount() {
    local node=$1
    local dev=$2
    local dir=$3

    echo -n "Checking $dev mount on $node..."
    if do_node_mute $node "grep -q $dir' ' /proc/mounts"; then
        echo "OK!"
        return 0
    fi

    if ! do_node_mute $node "grep -q ^$dev /etc/fstab"; then
        my_do_node $node "echo '$dev $dir $dev defaults 0 0' >> /etc/fstab" || \
            return ${PIPESTATUS[0]}
    fi
    my_do_node $node "mkdir -p $dir && mount $dir" || true

    if ! do_node_mute $node "grep -q $dir' ' /proc/mounts"; then
        echo "Failed to mount fs $dev at $dir!"
        return 1
    fi
    echo "OK!"
}

#
# configure nfsd mount on MDS and OSS nodes
#
cfg_nfs_mount() {
    local checked=""
    local node

    echo "+++ Configuring nfsd mount"

    for node in $MY_MGSNODE $MY_OSSNODES $MY_MDSNODES; do
        is_part_of $node $checked && continue
        cfg_mount $node nfsd /proc/fs/nfsd || return ${PIPESTATUS[0]}
        checked="$checked $node"
    done
}

get_pkgname() {
    local node=$1
    local pkg=$2

    my_do_node $node "rpm -q $pkg 2>&1" | tail -n1
    return ${PIPESTATUS[0]}
}

get_krb5pkgname() {
    local node=$1
    local flavor=$2

    my_do_node $node cat /etc/SuSE-release 2>/dev/null | \
    grep -q 'Enterprise Server 10'
    if [ ${PIPESTATUS[1]} -eq 0 ]; then
        case $flavor in
            cli) echo "krb5-client";;
            lib) echo "krb5";;
        esac
    else
        case $flavor in
            cli) echo "krb5-workstation";;
            lib) echo "krb5-libs";;
        esac
    fi
}

check_kdc() {
    local pkg
    local rc

    echo "+++ Checking KDC installation"

    echo -n "Checking $MY_KDCNODE..."
    pkg=$(get_pkgname $MY_KDCNODE $KRB5PKG_SVR)
    rc=${PIPESTATUS[0]}
    if [ $rc -ne 0 ]; then
        echo -e "\nCan not find $KRB5PKG_SVR package on $MY_KDCNODE: $pkg"
        return $rc
    fi
    echo "OK!"
}

check_krb5() {
    local checked=""
    local pkg
    local rc
    local krb5pkg_cli

    echo "+++ Checking Kerberos 5 installation"
    for node in $MY_MGSNODE $MY_OSSNODES $MY_MDSNODES $MY_CLIENTNODES; do
        is_part_of $node $checked && continue

        echo -n "Checking $node..."
        krb5pkg_cli=$(get_krb5pkgname $node cli)

        pkg=$(get_pkgname $node $krb5pkg_cli)
        rc=${PIPESTATUS[0]}
        if [ $rc -ne 0 ]; then
            echo -e "\nCan not find $krb5pkg_cli package on $node: $pkg"
            return $rc
        fi
        echo "OK!"
        checked="$checked $node"
    done
}

check_libgssapi() {
    local checked=""
    local node
    local pkg
    local rc

    echo "+++ Checking libgssapi installation"

    LIBGSSAPI=$(get_pkgname $MY_KDCNODE libgssapi)
    rc=${PIPESTATUS[0]}
    if [ $rc -ne 0 ]; then
        echo "Can not find libgssapi package on $MY_KDCNODE: $LIBGSSAPI"
        return $rc
    fi

    for node in $MY_MGSNODE $MY_OSSNODES $MY_MDSNODES $MY_CLIENTNODES; do
        is_part_of $node $checked && continue

        echo -n "Checking $node..."
        pkg=$(get_pkgname $node libgssapi)
        rc=${PIPESTATUS[0]}
        if [ $rc -ne 0 ]; then
            echo -e "\nCan not find libgssapi package on $node: $pkg"
            return $rc
        fi
        echo "OK!"
        checked="$checked $node"
    done
}

#
# check and update the /etc/gssapi_mech.conf file on each node
# We only support MIT Kerberos 5 GSS-API mechanism.
#
cfg_libgssapi() {
    local checked=""
    local node
    local pkg
    local rc
    local krb5pkg_lib
    local krb5_lib

    echo "+++ Updating $GSSAPI_MECH_CONF"

    for node in $MY_KDCNODE $MY_MGSNODE $MY_OSSNODES $MY_MDSNODES $MY_CLIENTNODES
    do
        is_part_of $node $checked && continue

        krb5pkg_lib=$(get_krb5pkgname $node lib)
        pkg=$(get_pkgname $node $krb5pkg_lib)
        rc=${PIPESTATUS[0]}
        if [ $rc -ne 0 ]; then
            echo -e "\nCan not find $krb5pkg_lib package on $node: $pkg"
            return $rc
        fi

        krb5_lib=$(my_do_node $node "rpm -ql $pkg" | \
                    grep libgssapi_krb5.so | head -n1)

        if ! do_node_mute $node \
"egrep -q \\\"^$krb5_lib|^$(basename $krb5_lib)\\\" $GSSAPI_MECH_CONF"; then
            do_node_mute $node \
"echo '$krb5_lib mechglue_internal_krb5_init' >> $GSSAPI_MECH_CONF"
        fi
        checked="$checked $node"
    done
    echo "OK!"
}

#
# check and update the /etc/request-key.conf file on each MDS and client node
#
cfg_keyutils() {
    local checked=""
    local node
    local lgss_keyring

    echo "+++ Updating $REQUEST_KEY_CONF"

    for node in $MY_OSSNODES $MY_MDSNODES $MY_CLIENTNODES; do
        is_part_of $node $checked && continue
        lgss_keyring=$(my_do_node $node "which lgss_keyring") || \
            return ${PIPESTATUS[0]}

        if ! do_node_mute $node \
"grep -q \\\"^create.*$lgss_keyring\\\" $REQUEST_KEY_CONF"; then
            do_node_mute $node \
"echo 'create lgssc * * $lgss_keyring %o %k %t %d %c %u %g %T %P %S' \
>> $REQUEST_KEY_CONF"
        fi
        checked="$checked $node"
    done
    echo "OK!"
}

add_svc_princ() {
    local fqdn=$1
    local type=$2

    echo -n "Creating service principal lustre_$type/$fqdn@$KRB5_REALM..."
    do_kdc_mute "kadmin.local -r $KRB5_REALM <<EOF
addprinc -randkey lustre_$type/$fqdn@$KRB5_REALM
EOF"
    local rc=${PIPESTATUS[0]}
    [ $rc -ne 0 ] && echo "Failed!" || echo "OK!"

    return $rc
}

add_svc_princ_root() {
    echo -n "Creating service principal lustre_root@$KRB5_REALM..."
    do_kdc_mute "kadmin.local -r $KRB5_REALM <<EOF
addprinc -randkey lustre_root@$KRB5_REALM
EOF"
    local rc=${PIPESTATUS[0]}
    [ $rc -ne 0 ] && echo "Failed!" || echo "OK!"

    return $rc
}

add_user_princ() {
    local user=$1

    echo -n "Creating user principal $user@$KRB5_REALM..."
    do_kdc_mute "kadmin.local -r $KRB5_REALM <<EOF
addprinc -pw $user $user@$KRB5_REALM
EOF"
    local rc=${PIPESTATUS[0]}
    [ $rc -ne 0 ] && echo "Failed!" || echo "OK!"

    return $rc
}

add_test_princ_id() {
    local id=$1
    local user

    user=$(do_kdc getent passwd $id | cut -d: -f1)
    if [ -z "$user" ]; then
        echo "Can not find the user with uid $id on the KDC!"
        return 1
    fi

    add_user_princ $user || return ${PIPESTATUS[0]}
}

#
# create principals for the client, MDS, OSS, runas users and add them to
# the Kerberos database
#
cfg_kdc_princs() {
    local node

    add_svc_princ $MY_MGSNODE mgs || return ${PIPESTATUS[0]}

    for node in $MY_MDSNODES; do
        add_svc_princ $node mds || return ${PIPESTATUS[0]}
    done

    for node in $MY_OSSNODES; do
        add_svc_princ $node oss || return ${PIPESTATUS[0]}
    done

    for node in $MY_CLIENTNODES; do
        if $SPLIT_KEYTAB; then
            add_svc_princ $node root || return ${PIPESTATUS[0]}
        else
            add_svc_princ_root || return ${PIPESTATUS[0]}
        fi
    done

    if ! $SPLIT_KEYTAB; then
        add_user_princ lustre_root || return ${PIPESTATUS[0]}
    fi
    add_user_princ bin || return ${PIPESTATUS[0]}
    add_user_princ daemon || return ${PIPESTATUS[0]}
    add_user_princ games || return ${PIPESTATUS[0]}

    if $CFG_RUNAS; then
        for uid in $LOCAL_UIDS; do
            add_test_princ_id $uid || return ${PIPESTATUS[0]}
        done
    fi
}

#
# create and install the KDC configuration file kdc.conf on the KDC, which
# will destroy the old KDC setting
#
cfg_kdc() {
    local tmpdir="$TMP/krb5_cfg_tmp_$UID"
    local tmpcfg=$tmpdir/kdc.conf
    local tmpacl=$tmpdir/kadm5.acl

    echo "+++ Configuring KDC on $MY_KDCNODE"
    echo "Warning: old KDC setting on $MY_KDCNODE will be destroied!!!"

    echo -n "Checking the existence of KDC config dir..."
    do_kdc_mute "[ -d $KDC_CONF_DIR ]"
    if [ ${PIPESTATUS[0]} -ne 0 ]; then
        echo -e "\nUnrecognized krb5 distribution!"
        return 1
    else
        echo "OK!"
    fi

    # stop KDC daemon
    do_kdc_mute "/etc/init.d/krb5kdc stop < /dev/null" || true

    echo -n "Removing old KDC configurations..."
    do_kdc_mute "rm -f $KDC_CONF_DIR/*"
    echo "OK!"

    # create kdc.conf locally
    rm -rf $tmpdir
    mkdir -p $tmpdir || return ${PIPESTATUS[0]}
    cat <<EOF > $tmpcfg
[kdcdefaults]
 acl_file = $KDC_ACL

[realms]
 $KRB5_REALM = {
  master_key_type = aes128-cts
  supported_enctypes = des3-hmac-sha1:normal aes128-cts:normal aes256-cts:normal des-cbc-md5:normal
 }
EOF

    # install kdc.conf remotely
    echo -n "Installing kdc.conf on $MY_KDCNODE..."
    $SCP $tmpcfg root@$MY_KDCNODE:$KDC_CONF || return ${PIPESTATUS[0]}
    echo "OK!"

    # initialize KDC database
    echo -n "Creating Kerberos database on $MY_KDCNODE..."
    do_kdc_mute "kdb5_util create -r $KRB5_REALM -s -P 111111"
    local rc=${PIPESTATUS[0]}
    if [ $rc -ne 0 ]; then
        echo "Failed!"
        return $rc
    else
        echo "OK!"
    fi

    # create ACL file locally & install remotely
    cat <<EOF > $tmpacl
*/admin@$KRB5_REALM   *
root@$KRB5_REALM      *
EOF
    echo -n "Installing kadm5.acl on $MY_KDCNODE..."
    $SCP $tmpacl root@$MY_KDCNODE:$KDC_ACL || return ${PIPESTATUS[0]}
    echo "OK!"
    rm -rf $tmpdir || true

    # start KDC daemon
    do_kdc "/etc/init.d/krb5kdc restart < /dev/null" || return ${PIPESTATUS[0]}
}

#
# create and install the Kerberos configuration file krb5.conf on the KDC,
# client, MDS and OSS
#
cfg_krb5_conf() {
    local tmpdir="$TMP/krb5_cfg_tmp_$UID"
    local tmpcfg="$tmpdir/krb5.conf"
    local checked=""

    echo "+++ Installing krb5.conf on all nodes"

    # create krb5.conf locally
    rm -rf $tmpdir
    mkdir -p $tmpdir || return ${PIPESTATUS[0]}
    cat <<EOF > $tmpcfg
[libdefaults]
 default_realm = $KRB5_REALM
 dns_lookup_realm = false
 dns_lookup_kdc = false
 ticket_lifetime = $KRB5_TICKET_LIFETIME
 forwardable = yes

[realms]
 $KRB5_REALM = {
  kdc = $MY_KDCNODE:88
  admin_server = $MY_KDCNODE:749
  default_domain = $KRB5_DOMAIN
 }

[domain_realm]
 .$KRB5_DOMAIN = $KRB5_REALM
 $KRB5_DOMAIN = $KRB5_REALM

[kdc]
 profile = $KDC_CONF

[appdefaults]
 pam = {
  debug = false
  forwardable = true
  krb4_convert = false
 }
EOF

    # install krb5.conf remotely
    for node in $MY_KDCNODE $MY_MGSNODE $MY_OSSNODES $MY_MDSNODES $MY_CLIENTNODES
    do
        is_part_of $node $checked && continue

        echo -n "Installing krb5.conf on $node..."
        $SCP $tmpcfg root@$node:$KRB5_CONF || return ${PIPESTATUS[0]}
        echo "OK!"

        checked="$checked $node"
    done
    rm -rf $tmpdir || true
}

add_keytab() {
    local tab=$1
    local princ=$2
    local enctype=$3

    do_kdc_mute "kadmin.local -r $KRB5_REALM <<EOF
ktadd -k $tab -e $enctype:normal $princ@$KRB5_REALM
EOF"
}

add_keytab_svc() {
    local tab=$1
    local fqdn=$2
    local type=$3
    local enctype=$4

    add_keytab $tab lustre_$type/$fqdn $enctype
}

add_keytab_root() {
    local tab=$1
    local enctype=$2

    add_keytab $tab lustre_root $enctype
}

merge_keytab() {
    local tab=$1
    local node=$2

    $SCP $tab root@$node:$tab || return ${PIPESTATUS[0]}
    do_node_mute $node "ktutil <<EOF
rkt $tab
wkt $KRB5_KEYTAB
EOF" || return ${PIPESTATUS[0]}
}

#
# create and install the keytab file krb5.keytab on the client, MDS and OSS
#
cfg_keytab() {
    local tmptab="$TMP/keytab.tmp"
    local node

    echo "+++ Generating keytabs"

    # remove old keytabs
    echo -n "Deleting old keytabs on all nodes..."
    for node in $MY_MGSNODE $MY_OSSNODES $MY_MDSNODES $MY_CLIENTNODES; do
        do_node_mute $node "rm -f $KRB5_KEYTAB $TMP/krb5cc*"
    done
    echo "OK!"

    # install for MDS nodes
    for node in $MY_MDSNODES; do
        echo -n "Preparing for MDS $node..."
        do_kdc_mute "rm -f $tmptab"
        add_keytab_svc $tmptab $node mds $MDS_ENCTYPE || return ${PIPESTATUS[0]}

        if is_part_of $node $MY_MGSNODE; then
            echo -n "also be an MGS..."
            add_keytab_svc $tmptab $node mgs $MGS_ENCTYPE || \
                return ${PIPESTATUS[0]}
        fi

        if is_part_of $node $MY_OSSNODES; then
            echo -n "also be an OSS..."
            add_keytab_svc $tmptab $node oss $OSS_ENCTYPE || \
                return ${PIPESTATUS[0]}
        fi
        echo "OK!"

        echo -n "Installing krb5.keytab on $node..."
        $SCP root@$MY_KDCNODE:$tmptab $tmptab || return ${PIPESTATUS[0]}
        $SCP $tmptab root@$node:$KRB5_KEYTAB || return ${PIPESTATUS[0]}
        echo "OK!"
        rm -f $tmptab
    done

    # install for MGS node
    echo -n "Preparing for MGS $MY_MGSNODE..."
    if ! is_part_of $MY_MGSNODE $MY_MDSNODES; then
        do_kdc_mute "rm -f $tmptab"
        add_keytab_svc $tmptab $MY_MGSNODE mgs $MGS_ENCTYPE || \
            return ${PIPESTATUS[0]}

        if is_part_of $MY_MGSNODE $MY_OSSNODES; then
            echo -n "also be an OSS..."
            add_keytab_svc $tmptab $MY_MGSNODE oss $OSS_ENCTYPE || \
                return ${PIPESTATUS[0]}
        fi
        echo "OK!"

        echo -n "Installing krb5.keytab on $MY_MGSNODE..."
        $SCP root@$MY_KDCNODE:$tmptab $tmptab || return ${PIPESTATUS[0]}
        $SCP $tmptab root@$MY_MGSNODE:$KRB5_KEYTAB || return ${PIPESTATUS[0]}
        echo "OK!"
        rm -f $tmptab
    else
        echo "also be an MDS, already done, skip"
    fi

    # install for OSS nodes
    for node in $MY_OSSNODES; do
        echo -n "Preparing for OSS $node..."
        if is_part_of $node $MY_MDSNODES; then
            echo "also be an MDS, already done, skip"
        elif is_part_of $node $MY_MGSNODE; then
            echo "also be an MGS, already done, skip"
        else
            do_kdc_mute "rm -f $tmptab"
            add_keytab_svc $tmptab $node oss $OSS_ENCTYPE || \
                return ${PIPESTATUS[0]}
            echo "OK!"

            echo -n "Installing krb5.keytab on $node..."
            $SCP root@$MY_KDCNODE:$tmptab $tmptab || return ${PIPESTATUS[0]}
            $SCP $tmptab root@$node:$KRB5_KEYTAB || return ${PIPESTATUS[0]}
            echo "OK!"
            rm -f $tmptab
        fi
    done

    # install for client nodes
    do_kdc_mute "rm -f $tmptab"
    if ! $SPLIT_KEYTAB; then
        echo -n "Preparing for client..."
        add_keytab_root $tmptab $CLIENT_ENCTYPE || return ${PIPESTATUS[0]}
        $SCP root@$MY_KDCNODE:$tmptab $tmptab || return ${PIPESTATUS[0]}
        echo "OK!"
    else
        for node in $MY_CLIENTNODES; do
            echo -n "Preparing for client $node..."
            # don't generate keytabs if it's also an MDS
            if is_part_of $node $MY_MDSNODES; then
                echo "also be an MDS, already done, skip"
                continue
            fi

            add_keytab_svc $tmptab $node root $CLIENT_ENCTYPE || \
                return ${PIPESTATUS[0]}
            $SCP root@$MY_KDCNODE:$tmptab $tmptab || return ${PIPESTATUS[0]}
            echo "OK!"
        done
    fi
    for node in $MY_CLIENTNODES; do
        echo -n "Installing krb5.keytab on client $node..."

        # don't install if it's also an MDS
        if is_part_of $node $MY_MDSNODES; then
            echo "also be an MDS, already done, skip"
            continue
        fi

        # merge keytab if it's also an MGS
        if is_part_of $node $MY_MGSNODE; then
            echo -n "also be an MGS, merging keytab..."
            merge_keytab $tmptab $node || return ${PIPESTATUS[0]}
            echo "OK!"
            continue
        fi

        # merge keytab if it's also an OSS
        if is_part_of $node $MY_OSSNODES; then
            echo -n "also be an OSS, merging keytab..."
            merge_keytab $tmptab $node || return ${PIPESTATUS[0]}
            echo "OK!"
            continue
        fi

        # simply install otherwise
        $SCP $tmptab root@$node:$KRB5_KEYTAB || return ${PIPESTATUS[0]}
        echo "OK!"
    done
    rm -f $tmptab || true
}

check_acceptor_port() {
    local node=$1
    local port=$2

    if [ -z "$port" ]; then
        echo "Missing acceptor port!"
        return 1
    fi

    local WAIT=0
    local MAX_WAIT=300
    while [ $WAIT -lt $MAX_WAIT ]; do
        sleep 5
        my_do_node $node "netstat -tpan" | grep -q ":$port .*TIME_WAIT"
        if [ ${PIPESTATUS[1]} -ne 0 ]; then
            return 0
        fi
        WAIT=$((WAIT + 5))
    done

    echo "LNET acceptor port $port is in use on node $node!"
    return 2
}

get_client_nids() {
    local client_nids=""
    local node
    local nid
    local local_fqdn
    local rc

    # get the fqdn of the local host
    local_fqdn=$(get_fqdn $HOSTNAME)
    rc=${PIPESTATUS[0]}
    if [ $rc -ne 0 ]; then
        echo $local_fqdn
        return $rc
    fi

    for node in $MY_CLIENTNODES; do
        my_do_node $node lsmod | grep -q lnet || \
        my_do_node $node "modprobe lnet" || {
            if [ "$node" = "$local_fqdn" ]; then
                lsmod | grep -q lnet || load_modules
            else
                echo "Failed to load lnet module on node $node!"
                return 1
            fi
        }

        check_acceptor_port $node $ACCEPTOR_PORT || return ${PIPESTATUS[0]}

        nid=$(set +x; my_do_node $node \
"$LCTL net up 1>/dev/null && $LCTL list_nids" 2>&1 | head -n1
exit ${PIPESTATUS[0]})
        rc=${PIPESTATUS[0]}
        if [ $rc -ne 0 ]; then
            echo "Failed to get the nid for node $node: $nid"
            return $rc
        fi
        [ -z "$client_nids" ] && client_nids="$nid" \
        || client_nids="$client_nids $nid"

        my_do_node $node "$LCTL net down 1>/dev/null" || true
    done

    echo "$client_nids"
    return 0
}

# ******************************** Main Flow ******************************** #
normalize_names || exit ${PIPESTATUS[0]}
check_rsh || exit ${PIPESTATUS[0]}
check_entropy || exit ${PIPESTATUS[0]}

if $CFG_RUNAS; then
    check_users || exit ${PIPESTATUS[0]}
fi

check_kdc || exit ${PIPESTATUS[0]}
check_krb5 || exit ${PIPESTATUS[0]}
check_libgssapi || exit ${PIPESTATUS[0]}

echo "===================================================================="
echo " Configure Kerberos testing environment for Lustre"
echo " KDC: $MY_KDCNODE"
echo " realm: $KRB5_REALM, domain: $KRB5_DOMAIN"
echo " Using gssapi package: $LIBGSSAPI"
echo " MGS node:"
echo "     $MY_MGSNODE"
echo " OSS nodes:"
for i in $MY_OSSNODES; do echo "     $i"; done
echo " MDS nodes:"
for i in $MY_MDSNODES; do echo "     $i"; done
echo " CLIENT nodes:"
for i in $MY_CLIENTNODES; do echo "     $i"; done
echo "===================================================================="

cfg_nfs_mount || exit ${PIPESTATUS[0]}
cfg_libgssapi || exit ${PIPESTATUS[0]}
cfg_keyutils || exit ${PIPESTATUS[0]}

if $RESET_KDC; then
    cfg_krb5_conf || exit ${PIPESTATUS[0]}
    cfg_kdc || exit ${PIPESTATUS[0]}
fi

cfg_kdc_princs || exit ${PIPESTATUS[0]}
cfg_keytab || exit ${PIPESTATUS[0]}

echo "Complete successfully!"


#!/bin/bash
#set -x
set -e

LUSTRE=${LUSTRE:-`dirname $0`/..}
. $LUSTRE/tests/test-framework.sh
init_test_env $@
. ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
init_logging

# QE uses the following parameters:
# size=128 crghi=16 thrhi=32
crghi=${crghi:-2}
thrhi=${thrhi:-16}
size=${size:-1024}

[ "$SLOW" = no ] && { crghi=2; thrhi=2; }

if [ "$SGPDD_YES" != "yes" -o "$REFORMAT" != "yes" ]; then
	skip_env "$0 reformats all devices, set SGPDD_YES=yes REFORMAT=yes"
	exit 0
fi

# Skip these tests
ALWAYS_EXCEPT="$SGPDD_SURVEY_EXCEPT"

build_test_filter

init_facets_vars

cleanupall

run_sgpdd_host () {
    local host=$1
    local devs=$2

    local params="size=$size crghi=$crghi thrhi=$thrhi"
    do_rpc_nodes $host run_sgpdd $devs "$params"
}

run_sgpdd_facets () {
    local facets=$1
    local facet

    local facetshosts
    for facet in ${facets//,/ }; do
        local host=$(facet_host $facet)
        local dev=${facet}_dev
        local var=$(node_var_name ${host}_devs)
        eval ${var}=$(expand_list ${!var} ${!dev})
        facetshosts=$(expand_list $facetshosts $host)
    done

    for host in ${facetshosts//,/ }; do
        var=$(node_var_name ${host}_devs)
        echo "=== $facets === $host === ${!var} ==="
        local scsidevs=${!var}
        run_sgpdd_host $host ${scsidevs}
    done
}

test_1 () {
	local mdss=$(get_facets MDS)

	check_progs_installed $(facets_hosts $mdss) $SGPDDSURVEY sg_map ||
		skip_env "SGPDDSURVEY=$SGPDDSURVEY or sg_map not found"

	run_sgpdd_facets $mdss
}
run_test 1 "sgpdd-survey, mds, scsidevs"

test_2 () {
	local osts=$(get_facets OST)

	check_progs_installed $(facets_hosts $osts) $SGPDDSURVEY sg_map ||
		skip_env "SGPDDSURVEY=$SGPDDSURVEY or sg_map not found"

	run_sgpdd_facets $osts
}
run_test 2 "sgpdd-survey, osts, scsidevs"

complete $SECONDS
exit_status