#!/bin/bash
#
# Run select tests by setting ONLY, or as arguments to the script.
# Skip specific tests by setting EXCEPT.
#
set -e

ONLY=${ONLY:-"$*"}

KILL=/bin/kill
LREPL_LOG=$TMP/lustre_rsync.log
ORIG_PWD=${PWD}

LUSTRE=${LUSTRE:-$(dirname $0)/..}
. $LUSTRE/tests/test-framework.sh
init_test_env $@
init_logging

ALWAYS_EXCEPT="$LRSYNC_EXCEPT "
# bug number for skipped test: LU-4256
ALWAYS_EXCEPT+="               2b"
# UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!

build_test_filter

[ -n "$FILESET" ] && skip "Not functional for FILESET set"

check_and_setup_lustre

DIR=${DIR:-$MOUNT}
assert_DIR

if getent group nobody; then
	GROUP=nobody
elif getent group nogroup; then
	GROUP=nogroup
else
	error "No generic nobody group"
fi

export LRSYNC=${LRSYNC:-"$LUSTRE/utils/lustre_rsync"}
[ ! -f "$LRSYNC" ] && export LRSYNC=$(which lustre_rsync)
export LRSYNC="$LRSYNC -v -c no -d 2"

# Number of seconds to run dbench
DBENCH_TIME=${DBENCH_TIME:-60}
TGT=$TMP/target
TGT2=$TMP/target2
MDT0=$($LCTL get_param -n mdc.*.mds_server_uuid |
	awk '{ gsub(/_UUID/,""); print $1 }' | head -n1)

init_changelog() {
	changelog_register || error "changelog_register failed"
	CL_USER=(${CL_USERS[$SINGLEMDS]})
	echo $MDT0: Registered changelog user $CL_USER
	[ -z $CL_USER ] &&
		echo "No changelog users present on $SINGLEMDS"
}

fini_changelog() {
	changelog_clear
	changelog_deregister
}

init_src() {
	rm -rf $TGT/$tdir $TGT/d*.lustre_rsync-test 2> /dev/null
	rm -rf $TGT2/$tdir $TGT2/d*.lustre_rsync-test 2> /dev/null
	rm -rf ${DIR}/$tdir $DIR/d*.lustre_rsync-test ${DIR}/tgt 2> /dev/null
	rm -f $LREPL_LOG
	mkdir -p ${DIR}/$tdir || error "Failed to create target: " $DIR/$tdir
	mkdir -p ${TGT}/$tdir || error "Failed to create target: " $TGT/$tdir
	mkdir -p ${TGT2}/$tdir || error "Failed to create target: " $TGT2/$tdir
}

cleanup_src_tgt() {
	rm -rf $TGT/$tdir
	rm -rf $DIR/$tdir
	rm -rf $DIR/tgt
}

# Check whether the filesystem supports xattr or not.
# Return value:
# "large" - large xattr is supported
# "small" - large xattr is unsupported but small xattr is supported
# "no"    - xattr is unsupported
check_xattr() {
    local tgt=$1
    local xattr="no"

    touch $tgt

    local val="$(generate_string $(max_xattr_size))"
    if large_xattr_enabled &&
       setfattr -n user.foo -v $val $tgt 2>/dev/null; then
            xattr="large"
    else
        setfattr -n user.foo -v bar $tgt 2>/dev/null && xattr="small"
    fi

    rm -f $tgt
    echo $xattr
}

check_diff() {
	local changelog_file=$(generate_logname "changelog")

	if [ -e $1 -o -e $2 ]; then
		diff -rq -x "dev1" $1 $2
		local RC=$?
		if [ $RC -ne 0 ]; then
			$LFS changelog $MDT0 > $changelog_file
			error "Failure in replication; differences found."
		fi
	fi
}

procs_are_stopped() {
	local pids="$*"
	local state

	for state in $(ps -p "$pids" -o state=); do
		if [[ "$state" != T ]]; then
			return 1
		fi
	done

	return 0
}

# Send SIGSTOP to PIDs and wait up to 60 seconds for them to show a
# stopped process state.
stop_procs() {
	local pids="$*"
	local end

	$KILL -SIGSTOP $pids
	end=$((SECONDS + 60))
	while ((SECONDS < end)); do
		if procs_are_stopped $pids; then
			return 0
		fi

		sleep 1
	done

	return 1
}

# Test 1A - test basic operations
test_1A() { # was test_1
    init_src
    init_changelog
    local xattr=$(check_xattr $TGT/foo)

    # Directory create
    mkdir $DIR/$tdir/d1
    mkdir $DIR/$tdir/d2

    # File create
    touch $DIR/$tdir/file1
    cp /etc/hosts  $DIR/$tdir/d1/
    touch  $DIR/$tdir/d1/"space in filename"
    touch  $DIR/$tdir/d1/file2

    # File rename
    mv $DIR/$tdir/d1/file2 $DIR/$tdir/d2/file3

    # File and directory delete
    touch $DIR/$tdir/d1/file4
    mkdir $DIR/$tdir/d1/del
    touch  $DIR/$tdir/d1/del/del1
    touch  $DIR/$tdir/d1/del/del2
    rm -rf $DIR/$tdir/d1/del
    rm $DIR/$tdir/d1/file4

    #hard and soft links
    cat /etc/hosts > $DIR/$tdir/d1/link1
    ln  $DIR/$tdir/d1/link1  $DIR/$tdir/d1/link2
    ln -s $DIR/$tdir/d1/link1  $DIR/$tdir/d1/link3

    # Device files
    #mknod $DIR/$tdir/dev1 b 8 1

	# Replicate
	local LRSYNC_LOG=$(generate_logname "lrsync_log")
	echo "Replication #1"
	$LRSYNC -s $DIR -t $TGT -t $TGT2 -m $MDT0 -u $CL_USER -l $LREPL_LOG \
		-D $LRSYNC_LOG

    # Set attributes
    chmod 000 $DIR/$tdir/d2/file3
    chown nobody:$GROUP $DIR/$tdir/d2/file3

    # Set xattrs
    if [[ "$xattr" != "no" ]]; then
        local value
        touch $DIR/$tdir/file5
        [[ "$xattr" = "large" ]] &&
            value="$(generate_string $(max_xattr_size))" || value="bar"
        setfattr -n user.foo -v $value $DIR/$tdir/file5
    fi

	echo "Replication #2"
	$LRSYNC -l $LREPL_LOG -D $LRSYNC_LOG

	if [[ "$xattr" != "no" ]]; then
		local xval1=$(get_xattr_value user.foo $TGT/$tdir/file5)
		local xval2=$(get_xattr_value user.foo $TGT2/$tdir/file5)

		if [[ "$xval1" != "$value" || "$xval2" != "$value" ]]; then
			error "Error in replicating xattrs."
		fi
	fi

	# Use diff to compare the source and the destination
	check_diff $DIR/$tdir $TGT/$tdir
	check_diff $DIR/$tdir $TGT2/$tdir

	fini_changelog
	cleanup_src_tgt
}
run_test 1A "Simple Replication"

# Test 1a - test create/delete operations in ROOT directory
test_1a() { # LU-5005
	rm -rf $TGT/root-* 2> /dev/null
	rm -rf $DIR/root-* 2> /dev/null
	init_src
	init_changelog

	# Directory create
	mkdir $DIR/root-dir

	# File create
	touch $DIR/root-file
	touch $DIR/root-file2

	# File rename
	mv $DIR/root-file2 $DIR/root-file3

	# File and directory delete
	touch $DIR/root-file4
	mkdir $DIR/root-dir1
	rm $DIR/root-file4
	rm -rf $DIR/root-dir1

	# Replicate
	local LRSYNC_LOG=$(generate_logname "lrsync_log")
	echo "Replication"
	$LRSYNC -s $DIR -t $TGT -m $MDT0 -u $CL_USER -l $LREPL_LOG \
		-D $LRSYNC_LOG

	# Verify
	stat $TGT/root-dir || error "Dir create not replicated"
	stat $TGT/root-file || error "File create not replicated"
	stat $TGT/root-file2 && error "Rename not replicated (src)"
	stat $TGT/root-file3 || error "Rename not replicated (tgt)"
	stat $TGT/root-dir1 && error "Dir delete not replicated"
	stat $TGT/root-file4 && error "File delete not replicated"

	cleanup_src_tgt
	fini_changelog
	rm -fr $TGT/root-*
	rm -fr $DIR/root-*
	return 0
}
run_test 1a "Replicate create/delete operations in ROOT directory"

# Test 2a - Replicate files created by dbench
test_2a() {
	init_src
	init_changelog

	# Run dbench
	sh rundbench -C -D $DIR/$tdir 2 -t $DBENCH_TIME || error "dbench failed"

	local LRSYNC_LOG=$(generate_logname "lrsync_log")
	# Replicate the changes to $TGT
	$LRSYNC -s $DIR -t $TGT -t $TGT2 -m $MDT0 -u $CL_USER -l $LREPL_LOG \
		-D $LRSYNC_LOG

	# Use diff to compare the source and the destination
	check_diff $DIR/$tdir $TGT/$tdir
	check_diff $DIR/$tdir $TGT2/$tdir

	fini_changelog
	cleanup_src_tgt
	return 0
}
run_test 2a "Replicate files created by dbench."

# Test 2b - Replicate files changed by dbench.
test_2b() {
	local child_pid
	init_src
	init_changelog

	# Run dbench
	sh rundbench -C -D $DIR/$tdir 2 -t $DBENCH_TIME &
	# wait for dbench to start
	wait_for_function 'child_pid=$(pgrep dbench)' 360
	# let dbench run for a bit
	sleep 10

	echo PIDs: $child_pid
	echo Stopping dbench
	stop_procs $child_pid

	local LRSYNC_LOG=$(generate_logname "lrsync_log")
	echo Starting replication
	$LRSYNC -s $DIR -t $TGT -t $TGT2 -m $MDT0 -u $CL_USER -l $LREPL_LOG \
		-D $LRSYNC_LOG
	check_diff $DIR/$tdir $TGT/$tdir

    echo Resuming dbench
    $KILL -SIGCONT $child_pid
    sleep 10

    echo Stopping dbench
	stop_procs $child_pid

	echo Starting replication
	$LRSYNC -l $LREPL_LOG -D $LRSYNC_LOG
	check_diff $DIR/$tdir $TGT/$tdir

    echo "Wait for dbench to finish"
    $KILL -SIGCONT $child_pid
    wait

	# Replicate the changes to $TGT
	echo Starting replication
	$LRSYNC -l $LREPL_LOG -D $LRSYNC_LOG

	check_diff $DIR/$tdir $TGT/$tdir
	check_diff $DIR/$tdir $TGT2/$tdir

    fini_changelog
    cleanup_src_tgt
    return 0
}
run_test 2b "Replicate files changed by dbench."

# Test 2c - Replicate files while dbench is running
test_2c() {
	init_src
	init_changelog

	# Run dbench
	sh rundbench -C -D $DIR/$tdir 2 -t $DBENCH_TIME &

	local LRSYNC_LOG=$(generate_logname "lrsync_log")
	# Replicate the changes to $TGT
	sleep 10 # give dbench a headstart
	local quit=0
	while [ $quit -le 1 ];
	do
		echo "Running lustre_rsync"
		$LRSYNC -s $DIR -t $TGT -t $TGT2 -m ${mds1_svc} -u $CL_USER \
			-l $LREPL_LOG -D $LRSYNC_LOG
		sleep 5
		pgrep dbench
		if [ $? -ne 0 ]; then
			quit=$(expr $quit + 1)
		fi
	done

	# Use diff to compare the source and the destination
	check_diff $DIR/$tdir $TGT/$tdir
	check_diff $DIR/$tdir $TGT2/$tdir

	fini_changelog
	cleanup_src_tgt
	return 0
}
run_test 2c "Replicate files while dbench is running."

# Test 3a - Replicate files created by createmany
test_3a() {
	init_src
	init_changelog

	local numfiles=1000
	createmany -o $DIR/$tdir/$tfile $numfiles || error "createmany failed"

	local LRSYNC_LOG=$(generate_logname "lrsync_log")
	# Replicate the changes to $TGT
	$LRSYNC -s $DIR -t $TGT -t $TGT2 -m $MDT0 -u $CL_USER -l $LREPL_LOG \
		-D $LRSYNC_LOG
	check_diff $DIR/$tdir $TGT/$tdir
	check_diff $DIR/$tdir $TGT2/$tdir

	fini_changelog
	cleanup_src_tgt
	return 0
}
run_test 3a "Replicate files created by createmany"

# Test 3b - Replicate files created by writemany
test_3b() {
	init_src
	init_changelog

	local time=60
	local threads=5
	writemany -q -a $DIR/$tdir/$tfile $time $threads ||
		error "writemany failed"

	local LRSYNC_LOG=$(generate_logname "lrsync_log")
	# Replicate the changes to $TGT
	$LRSYNC -s $DIR -t $TGT -t $TGT2 -m $MDT0 -u $CL_USER -l $LREPL_LOG \
		-D $LRSYNC_LOG

	check_diff $DIR/$tdir $TGT/$tdir
	check_diff $DIR/$tdir $TGT2/$tdir

	fini_changelog
	cleanup_src_tgt
	return 0
}
run_test 3b "Replicate files created by writemany"

# Test 3c - Replicate files created by createmany/unlinkmany
test_3c() {
	init_src
	init_changelog

	local numfiles=1000
	createmany -o $DIR/$tdir/$tfile $numfiles || error "createmany failed"
	unlinkmany $DIR/$tdir/$tfile $numfiles || error "unlinkmany failed"

	local LRSYNC_LOG=$(generate_logname "lrsync_log")
	# Replicate the changes to $TGT
	$LRSYNC -s $DIR -t $TGT -t $TGT2 -m $MDT0  -u $CL_USER -l $LREPL_LOG \
		-D $LRSYNC_LOG
	check_diff $DIR/$tdir $TGT/$tdir
	check_diff $DIR/$tdir $TGT2/$tdir

	fini_changelog
	cleanup_src_tgt
	return 0
}
run_test 3c "Replicate files created by createmany/unlinkmany"

# Test 4 - Replicate files created by iozone
test_4() {
	which iozone > /dev/null 2>&1
	[ $? -ne 0 ] && skip "iozone not found"

	init_src
	init_changelog

	END_RUN_FILE=${DIR}/$tdir/run LOAD_PID_FILE=${DIR}/$tdir/pid \
		MOUNT=${DIR}/$tdir run_iozone.sh &
	sleep 30
	child_pid=$(pgrep iozone)
	stop_procs $child_pid

	local LRSYNC_LOG=$(generate_logname "lrsync_log")
	# Replicate the changes to $TGT
	$LRSYNC -s $DIR -t $TGT -t $TGT2 -m $MDT0  -u $CL_USER -l $LREPL_LOG \
		-D $LRSYNC_LOG
	check_diff $DIR/$tdir $TGT/$tdir
	check_diff $DIR/$tdir $TGT2/$tdir

    $KILL -SIGCONT $child_pid
    sleep 60
    $KILL -SIGKILL $(pgrep run_iozone.sh)
    $KILL -SIGKILL $(pgrep iozone)

    # After killing 'run_iozone.sh', process 'iozone' becomes the
    # child of PID 1. Hence 'wait' does not wait for it. Killing
    # iozone first, means more iozone processes are spawned off which
    # is not desirable. So, after sending a sigkill, the test goes
    # into a wait loop for iozone to cleanup and exit.
    wait
    while [ "$(pgrep "iozone")" != "" ];
    do
      ps -ef | grep iozone | grep -v grep
      sleep 1;
    done

	$LRSYNC -l $LREPL_LOG -D $LRSYNC_LOG
	check_diff $DIR/$tdir $TGT/$tdir
	check_diff $DIR/$tdir $TGT2/$tdir

    fini_changelog
    cleanup_src_tgt
    return 0
}
run_test 4 "Replicate files created by iozone"

# Test 5a - Stop / start lustre_rsync
test_5a() {
	init_src
	init_changelog

	NUMTEST=2000
	createmany -o $DIR/$tdir/$tfile $NUMTEST

	# Replicate the changes to $TGT
	local LRSYNC_LOG=$(generate_logname "lrsync_log")
	$LRSYNC -s $DIR -t $TGT -t $TGT2 -m $MDT0 -u $CL_USER -l $LREPL_LOG \
		-D $LRSYNC_LOG &
	local child_pid=$!
	sleep 30
	$KILL -SIGHUP $child_pid
	wait
	$LRSYNC -l $LREPL_LOG -D $LRSYNC_LOG

	check_diff $DIR/$tdir $TGT/$tdir
	check_diff $DIR/$tdir $TGT2/$tdir

	fini_changelog
	cleanup_src_tgt
	return 0
}
run_test 5a "Stop / start lustre_rsync"

# Test 5b - Kill / restart lustre_rsync
test_5b() {
	init_src
	init_changelog

	NUMTEST=2000
	createmany -o $DIR/$tdir/$tfile $NUMTEST

	# Replicate the changes to $TGT
	local LRSYNC_LOG=$(generate_logname "lrsync_log")
	$LRSYNC -s $DIR -t $TGT -t $TGT2 -m $MDT0 -u $CL_USER -l $LREPL_LOG \
		-D $LRSYNC_LOG &
	local child_pid=$!
	sleep 30
	$KILL -SIGKILL $child_pid
	wait
	$LRSYNC -l $LREPL_LOG -D $LRSYNC_LOG

	check_diff $DIR/$tdir $TGT/$tdir
	check_diff $DIR/$tdir $TGT2/$tdir

	fini_changelog
	cleanup_src_tgt
	return 0
}
run_test 5b "Kill / restart lustre_rsync"

# Test 6 - lustre_rsync large no of hard links
test_6() {
	init_src
	init_changelog

	local num_links=128
	local i

	touch $DIR/$tdir/link0
	for ((i = 1; i < num_links - 1; i++)); do
		ln $DIR/$tdir/link0 $DIR/$tdir/link$i
	done
	# create an extra hard link of src name ending with dest name
	ln $DIR/$tdir/link0 $DIR/$tdir/ink0

	local LRSYNC_LOG=$(generate_logname "lrsync_log")
	# Replicate the changes to $TGT
	$LRSYNC -s $DIR -t $TGT -t $TGT2 -m $MDT0 -u $CL_USER -l $LREPL_LOG \
		-D $LRSYNC_LOG
	check_diff $DIR/$tdir $TGT/$tdir
	check_diff $DIR/$tdir $TGT2/$tdir

	local count1=$(stat --format=%h $TGT/$tdir/link0)
	local count2=$(stat --format=%h $TGT2/$tdir/link0)
	if ((count1 != num_links || count2 != num_links)); then
		ls -l $TGT/$tdir/link0 $TGT2/$tdir/link0
		error "Incorrect no of hard links found $count1, $count2"
	fi

	fini_changelog
	cleanup_src_tgt
	return 0
}
run_test 6 "lustre_rsync large no of hard links"

# Test 7 - lustre_rsync stripesize
test_7() {
	local numfiles=100

	init_src
	mkdir -p ${DIR}/tgt/$tdir
	init_changelog

	$LFS setstripe -c $OSTCOUNT $DIR/$tdir ||
		error "$LFS setstripe failed"
	createmany -o $DIR/$tdir/$tfile $numfiles

	# To simulate replication to another lustre filesystem, replicate
	# the changes to $DIR/tgt. We can't turn off the changelogs
	# while we are registered, so lustre_rsync better not try to
	# replicate the replication steps.  It seems ok :)

	local LRSYNC_LOG=$(generate_logname "lrsync_log")
	$LRSYNC -s $DIR -t $DIR/tgt -m $MDT0 -u $CL_USER -l $LREPL_LOG \
		-D $LRSYNC_LOG
	check_diff ${DIR}/$tdir $DIR/tgt/$tdir

	local i=0
	while [ $i -lt $numfiles ];
	do
		local count=$($LFS getstripe $DIR/tgt/$tdir/${tfile}$i |
			      awk '/stripe_count/ {print $2}')
		if [ $count -ne $OSTCOUNT ]; then
			error "Stripe size not replicated"
		fi
		i=$(expr $i + 1)
	done
	fini_changelog
	cleanup_src_tgt
	return 0
}
run_test 7 "lustre_rsync stripesize"

# Test 8 - Replicate multiple file/directory moves
test_8() {
    init_src
    init_changelog

    for i in 1 2 3 4 5 6 7 8 9; do
	mkdir $DIR/$tdir/d$i
	    for j in 1 2 3 4 5 6 7 8 9; do
		mkdir $DIR/$tdir/d$i/d$i$j
		createmany -o $DIR/$tdir/d$i/d$i$j/a 10 \
		    > /dev/null
		mv $DIR/$tdir/d$i/d$i$j $DIR/$tdir/d$i/d0$i$j
		createmany -o $DIR/$tdir/d$i/d0$i$j/b 10 \
	            > /dev/null
		mv $DIR/$tdir/d$i/d0$i$j/a0 $DIR/$tdir/d$i/d0$i$j/c0
	    done
	    mv $DIR/$tdir/d$i $DIR/$tdir/d0$i
    done

	local LRSYNC_LOG=$(generate_logname "lrsync_log")
	$LRSYNC -s $DIR -t $TGT -m $MDT0 -u $CL_USER -l $LREPL_LOG \
		-D $LRSYNC_LOG

	check_diff ${DIR}/$tdir $TGT/$tdir

    fini_changelog
    cleanup_src_tgt
    return 0
}
run_test 8 "Replicate multiple file/directory moves"

test_9() {
    init_src
    init_changelog

    mkdir $DIR/$tdir/foo
    touch $DIR/$tdir/foo/a1

	local LRSYNC_LOG=$(generate_logname "lrsync_log")
	$LRSYNC -s $DIR -t $TGT -m $MDT0 -u $CL_USER -l $LREPL_LOG \
		-D $LRSYNC_LOG

	check_diff ${DIR}/$tdir $TGT/$tdir

	rm -rf $DIR/$tdir/foo

	$LRSYNC -s $DIR -t $TGT -m $MDT0 -u $CL_USER -l $LREPL_LOG \
		-D $LRSYNC_LOG

	check_diff ${DIR}/$tdir $TGT/$tdir

    fini_changelog
    cleanup_src_tgt
    return 0
}
run_test 9 "Replicate recursive directory removal"

cd $ORIG_PWD
complete $SECONDS
check_and_cleanup_lustre
exit_status


#!/bin/sh

set -e

FILENAME=$1

if [ -r ~/.maloorc ] ; then
	source ~/.maloorc
else
	echo "Error: ~/.maloorc not found." \
	     "Please obtain this file from the maloo web interface," \
	     "under 'Upload results'"
	exit 1
fi

if [ -z $FILENAME ] ; then
	echo "Usage: ${0} <tarball or directory>"
	exit 2
fi

if [ ! -r $FILENAME ] ; then
	echo "Input file '$FILENAME' not found"
	exit 3
fi

echo Uploading $FILENAME to $MALOO_URL
if [ -d $FILENAME ] ; then
	pushd $FILENAME
	tar czf upload.tar.gz * |
		curl -F "user_id=${MALOO_USER_ID}" -F "upload=@upload.tar.gz" \
		     -F "user_upload_token=${MALOO_UPLOAD_TOKEN}" ${MALOO_URL} \
		     > /dev/null
	popd
else
	curl -F "user_id=${MALOO_USER_ID}" -F "upload=@${FILENAME}" \
	     -F "user_upload_token=${MALOO_UPLOAD_TOKEN}" ${MALOO_URL} \
	     > /dev/null
fi
echo Complete.


#!/bin/bash
set -e

LUSTRE=${LUSTRE:-$(dirname $0)/..}
. $LUSTRE/tests/test-framework.sh
init_test_env $@
init_logging

ALWAYS_EXCEPT="$MDS_SURVEY_EXCEPT "

build_test_filter

MDSSURVEY=${MDSSURVEY:-$(which mds-survey 2>/dev/null || true)}
[ -z ${MDSSURVEY} ] && skip_env "mds-survey not found"

if [ "$MDS1_VERSION" -lt $(version_code 2.3.51) ]; then
	skip "Need MDS version at least 2.3.51"
fi

file_count=${file_count:-150000}
dir_count=${dir_count:-4}
thrhi=${thrhi:-8}
thrlo=${thrlo:-1}

[ "$SLOW" = no ] && { file_count=50000; dir_count=2; thrhi=4; }

check_and_setup_lustre

adjust_inode() {
    local require_inode=0
    local avail_mdt_inode=0
    local avail_ost_inode=0

    require_inode=$((file_count * thrhi))
    # get available inode for mdt
    avail_mdt_inode=$(mdt_free_inodes 0)
    avail_mdt_inode=$((avail_mdt_inode * 9 / 10))

    # get available inode for ost
    for i in $($LFS df -i | grep ${FSNAME}-OST | awk '{print $4}'); do
        avail_ost_inode=$((avail_ost_inode + i))
    done
    avail_ost_inode=$((avail_ost_inode * 9 / 10))

    ((require_inode > avail_mdt_inode)) && require_inode=$avail_mdt_inode
    ((require_inode > avail_ost_inode)) && require_inode=$avail_ost_inode

    if ((require_inode == 0)); then
        error "Fail to get the inode count"
    fi
    # convert it back to per thread inode
    require_inode=$((require_inode / thrhi))

    echo $require_inode
}

file_count=$(adjust_inode)
ost_count=$($LCTL dl | grep -c osc)

# first unmount all the lustre clients
cleanup_mount $MOUNT
cleanup_mount $MOUNT2

get_target() {
	local node=$1
	local mdt
	local mdts
	local target

	mdts=$(do_node $node "$LCTL dl" |
	       awk "{if (\$2 == \"UP\" && \$3 == \"mdt\") {print \$4}}")

	for mdt in $mdts; do
		target+="${target:+ }$node:$mdt"
	done

	echo -n $target
}

get_targets() {
	local targets
	local node

	for node in $(mdts_nodes); do
		targets+="${targets:+ }$(get_target $node)"
	done

	echo -n $targets
}

mds_survey_run() {
	local layer=${1:-mdd}
	local stripe_count=${2:-0}
	local rc=0

	rm -f ${TMP}/mds_survey*

	local cmd="file_count=$file_count thrlo=$thrlo thrhi=$thrhi"
	cmd+=" dir_count=$dir_count layer=$layer stripe_count=$stripe_count"
	cmd+=" rslt_loc=${TMP} targets=\"$(get_targets)\" $MDSSURVEY"

	trap cleanup_echo_devs EXIT ERR

	echo + $cmd
	eval $cmd || rc=$?
	cat ${TMP}/mds_survey*
	rm -f ${TMP}/mds_survey*
	((rc == 0)) || error "mds-survey failed"
}

test_1() {
	mds_survey_run "mdd" "0"
}
run_test 1 "Metadata survey with zero-stripe"

test_2() {
	local mdscount=$(get_node_count "$(mdts_nodes)")

	[ $mdscount -gt 1 ] && skip_env "Only run this test on single MDS"
	[ $ost_count -eq 0 ] && skip_env "Need to mount OST to test"

	mds_survey_run "mdd" "1"
}
run_test 2 "Metadata survey with stripe_count = 1"

# remount the clients
restore_mount $MOUNT

complete $SECONDS
cleanup_echo_devs
check_and_cleanup_lustre
exit_status


#!/bin/bash
#
# This test was used in a set of CMD3 tests (cmd3-4 test).

LUSTRE=${LUSTRE:-`dirname $0`/..}
. $LUSTRE/tests/test-framework.sh
init_test_env $@
. ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}

assert_env CLIENTS MDSRATE SINGLECLIENT MPIRUN

MACHINEFILE=${MACHINEFILE:-$TMP/$(basename $0 .sh).machines}
BASEDIR=$MOUNT/mdsrate

# Requirements
# set NUM_FILES=0 to force TIME_PERIOD work
NUM_FILES=${NUM_FILES:-1000000}
TIME_PERIOD=${TIME_PERIOD:-600}                        # seconds

# Local test variables
TESTDIR_SINGLE="${BASEDIR}/single"
TESTDIR_MULTI="${BASEDIR}/multi"

LOG=${TESTSUITELOG:-$TMP/$(basename $0 .sh).log}
CLIENT=$SINGLECLIENT
NODES_TO_USE=${NODES_TO_USE:-$CLIENTS}
NUM_CLIENTS=$(get_node_count ${NODES_TO_USE//,/ })

[ ! -x ${MDSRATE} ] && error "${MDSRATE} not built."

log "===== $0 ====== "

check_and_setup_lustre

mkdir -p $BASEDIR
chmod 0777 $BASEDIR
mdsrate_STRIPEPARAMS=${mdsrate_STRIPEPARAMS:-${fs_STRIPEPARAMS:-"-c -1"}}
setstripe_getstripe $BASEDIR $mdsrate_STRIPEPARAMS

IFree=$(mdsrate_inodes_available)
if [ $IFree -lt $NUM_FILES ]; then
    NUM_FILES=$IFree
fi

generate_machine_file $NODES_TO_USE $MACHINEFILE || error "can not generate machinefile"

# Make sure we start with a clean slate
rm -f ${LOG}

if [ -n "$NOSINGLE" ]; then
    echo "NO Test for creates for a single client."
else
    # We can use np = $NUM_CLIENTS to speed up the cleanup
    mdsrate_cleanup $NUM_CLIENTS $MACHINEFILE $NUM_FILES $TESTDIR_SINGLE 'f%%d' --ignore

    log "===== $0 ### 1 NODE CREATE ###"

	COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --create --time ${TIME_PERIOD}
		--nfiles ${NUM_FILES} --dir ${TESTDIR_SINGLE} --filefmt 'f%%d'"
	echo "+ ${COMMAND}"
	mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} -np 1 ${COMMAND} |
		tee ${LOG}

	if [ ${PIPESTATUS[0]} != 0 ]; then
		[ -f $LOG ] && sed -e "s/^/log: /" $LOG
		error_noexit "mdsrate create on single client failed, aborting"
		mdsrate_cleanup $NUM_CLIENTS $MACHINEFILE $NUM_FILES \
				$TESTDIR_SINGLE 'f%%d' --ignore
		exit 1
	fi

    log "===== $0 ### 1 NODE UNLINK ###"

    COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --unlink
                --nfiles ${NUM_FILES} --dir ${TESTDIR_SINGLE} --filefmt 'f%%d'"
    echo "+ ${COMMAND}"
    mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} -np 1 ${COMMAND} | tee ${LOG}

    if [ ${PIPESTATUS[0]} != 0 ]; then
	[ -f $LOG ] && sed -e "s/^/log: /" $LOG
	error "mdsrate unlink on a single client failed, aborting"
    fi

    rmdir $TESTDIR_SINGLE
fi

IFree=$(mdsrate_inodes_available)
if [ $IFree -lt $NUM_FILES ]; then
    NUM_FILES=$IFree
fi

[ $NUM_CLIENTS -eq 1 ] && NOMULTI=yes
if [ -n "$NOMULTI" ]; then
    echo "NO test for create on multiple nodes."
else
    mdsrate_cleanup $NUM_CLIENTS $MACHINEFILE $NUM_FILES $TESTDIR_MULTI 'f%%d' --ignore

    log "===== $0 ### $NUM_CLIENTS NODES CREATE ###"

	COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --create --time ${TIME_PERIOD}
		--nfiles $NUM_FILES --dir ${TESTDIR_MULTI} --filefmt 'f%%d'"
	echo "+ ${COMMAND}"
	mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} -np ${NUM_CLIENTS} \
		${COMMAND} | tee ${LOG}

	if [ ${PIPESTATUS[0]} != 0 ]; then
		[ -f $LOG ] && sed -e "s/^/log: /" $LOG
		error_noexit "mdsrate create on multiple nodes failed, aborting"
		mdsrate_cleanup $NUM_CLIENTS $MACHINEFILE $NUM_FILES \
				$TESTDIR_MULTI 'f%%d' --ignore
		exit 1
	fi

    log "===== $0 ### $NUM_CLIENTS NODES UNLINK ###"

	COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --unlink
		--nfiles ${NUM_FILES} --dir ${TESTDIR_MULTI} --filefmt 'f%%d'"
	echo "+ ${COMMAND}"
	mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} -np ${NUM_CLIENTS} \
		${COMMAND} | tee ${LOG}

    if [ ${PIPESTATUS[0]} != 0 ]; then
	[ -f $LOG ] && sed -e "s/^/log: /" $LOG
	error "mdsrate unlink on multiple nodes failed, aborting"
    fi

    rmdir $TESTDIR_MULTI
fi

rmdir $BASEDIR || true
rm -f $MACHINEFILE
check_and_cleanup_lustre
#rm -f $LOG

exit 0


 #!/bin/bash
#
# This test was used in a set of CMD3 tests (cmd3-3 test).

LUSTRE=${LUSTRE:-`dirname $0`/..}
. $LUSTRE/tests/test-framework.sh
init_test_env $@
. ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}

assert_env CLIENTS MDSRATE SINGLECLIENT MPIRUN

MACHINEFILE=${MACHINEFILE:-$TMP/$(basename $0 .sh).machines}
BASEDIR=$MOUNT/mdsrate

# Requirements
NUM_FILES=${NUM_FILES:-1000000}
TIME_PERIOD=${TIME_PERIOD:-600}                        # seconds

# Local test variables
TESTDIR_SINGLE="${BASEDIR}/single"
TESTDIR_MULTI="${BASEDIR}/multi"

LOG=${TESTSUITELOG:-$TMP/$(basename $0 .sh).log}
CLIENT=$SINGLECLIENT
NODES_TO_USE=${NODES_TO_USE:-$CLIENTS}
NUM_CLIENTS=$(get_node_count ${NODES_TO_USE//,/ })
# XXX - this needs to be determined given the number of MDTs and the number
#       of clients.
THREADS_PER_CLIENT=3                   # threads/client for multi client test
if [ $NUM_CLIENTS -gt 50 ]; then
    THREADS_PER_CLIENT=1
fi

[ ! -x ${MDSRATE} ] && error "${MDSRATE} not built."

# Make sure we start with a clean slate
rm -f ${LOG}

log "===== $0 ====== "

check_and_setup_lustre

mkdir -p $BASEDIR
chmod 0777 $BASEDIR
mdsrate_STRIPEPARAMS=${mdsrate_STRIPEPARAMS:-${fs_STRIPEPARAMS:-"-i 0 -c 1"}}
setstripe_getstripe $BASEDIR $mdsrate_STRIPEPARAMS

IFree=$(mdsrate_inodes_available)
if [ $IFree -lt $NUM_FILES ]; then
    NUM_FILES=$IFree
fi

generate_machine_file $NODES_TO_USE $MACHINEFILE || error "can not generate machinefile"

if [ -n "$NOSINGLE" ]; then
    echo "NO Tests on single client."
else
    if [ -n "$NOCREATE" ]; then
        echo "NO Test for creates for a single client."
    else
        # We can use np = $NUM_CLIENTS to speed up the cleanup
        mdsrate_cleanup $NUM_CLIENTS $MACHINEFILE $NUM_FILES $TESTDIR_SINGLE 'f%%d' --ignore

        log "===== $0 ### 1 NODE CREATE ###"

	COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --create --time ${TIME_PERIOD}
		--nfiles $NUM_FILES --dir ${TESTDIR_SINGLE} --filefmt 'f%%d'"
	echo "+ ${COMMAND}"
	mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} -np 1 ${COMMAND} |
		tee ${LOG}

	if [ ${PIPESTATUS[0]} != 0 ]; then
		[ -f $LOG ] && sed -e "s/^/log: /" $LOG
		error_noexit "mdsrate create on single client failed, aborting"
		mdsrate_cleanup $NUM_CLIENTS $MACHINEFILE $NUM_FILES \
				$TESTDIR_SINGLE 'f%%d' --ignore
		exit 1
	fi
    fi

    if [ -n "$NOUNLINK" ]; then
        echo "NO Test for unlinks for a single client."
    else
        log "===== $0 ### 1 NODE UNLINK ###"

        COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --unlink
                     --nfiles ${NUM_FILES} --dir ${TESTDIR_SINGLE} --filefmt 'f%%d'"
        echo "+ ${COMMAND}"
        mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} -np 1 ${COMMAND} |
		tee ${LOG}

        if [ ${PIPESTATUS[0]} != 0 ]; then
            [ -f $LOG ] && sed -e "s/^/log: /" $LOG
            error "mdsrate unlinks for a single client failed, aborting"
        fi

        rmdir $TESTDIR_SINGLE
    fi
fi

IFree=$(mdsrate_inodes_available)
if [ $IFree -lt $NUM_FILES ]; then
    NUM_FILES=$IFree
fi

if [ -n "$NOMULTI" ]; then
    echo "NO tests on multiple nodes."
else
    if [ -n "$NOCREATE" ]; then
        echo "NO test for create on multiple nodes."
    else
        mdsrate_cleanup $NUM_CLIENTS $MACHINEFILE $NUM_FILES $TESTDIR_MULTI 'f%%d' --ignore

        log "===== $0 ### $NUM_CLIENTS NODES CREATE with $THREADS_PER_CLIENT threads per client ###"

	COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --create --time ${TIME_PERIOD}
		--nfiles $NUM_FILES --dir ${TESTDIR_MULTI} --filefmt 'f%%d'"
	echo "+ ${COMMAND}"
	mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
		-np $((NUM_CLIENTS * THREADS_PER_CLIENT)) ${COMMAND} |
		tee ${LOG}

	if [ ${PIPESTATUS[0]} != 0 ]; then
		[ -f $LOG ] && sed -e "s/^/log: /" $LOG
		error_noexit "mdsrate create on multiple nodes failed, aborting"
		mdsrate_cleanup $((NUM_CLIENTS * THREADS_PER_CLIENT)) \
				$MACHINEFILE $NUM_FILES \
				$TESTDIR_MULTI 'f%%d' --ignore
		exit 1
	fi
    fi

    if [ -n "$NOUNLINK" ]; then
        echo "NO Test for unlinks multiple nodes."
    else
        log "===== $0 ### $NUM_CLIENTS NODES UNLINK with $THREADS_PER_CLIENT threads per client ###"

        COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --unlink
                      --nfiles ${NUM_FILES} --dir ${TESTDIR_MULTI} --filefmt 'f%%d'"
        echo "+ ${COMMAND}"
        mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} \
		-np $((NUM_CLIENTS * THREADS_PER_CLIENT)) ${COMMAND} |
		tee ${LOG}
        if [ ${PIPESTATUS[0]} != 0 ]; then
            [ -f $LOG ] && sed -e "s/^/log: /" $LOG
            error "mdsrate unlinks multiple nodes failed, aborting"
        fi

        rmdir $TESTDIR_MULTI
    fi
fi

complete $SECONDS
rmdir $BASEDIR || true
rm -f $MACHINEFILE
check_and_cleanup_lustre
#rm -f $LOG

exit 0


#!/bin/bash
#
# This test was used in a set of CMD3 tests (cmd3-5 test).

# Directory lookup retrieval rate 10 directories 1 million files each
# 6000 random lookups/sec per client node 62,000 random lookups/sec aggregate
#
# In 10 dirs containing 1 million files each the mdsrate Test Program will
# perform lookups for 10 minutes. This test is run from a single node for
# #1 and from all nodes for #2 aggregate test to measure lookup performance.
# TEst performs lookups across all 10 directories.

LUSTRE=${LUSTRE:-`dirname $0`/..}
. $LUSTRE/tests/test-framework.sh
init_test_env $@
. ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
assert_env CLIENTS MDSRATE SINGLECLIENT MPIRUN

MACHINEFILE=${MACHINEFILE:-$TMP/$(basename $0 .sh).machines}
# Do not use name [df][0-9]* to avoid cleanup by rm, bug 18045
BASEDIR=$MOUNT/mdsrate

# Requirements
NUM_DIRS=${NUM_DIRS:-10}
NUM_FILES=${NUM_FILES:-1000000}
TIME_PERIOD=${TIME_PERIOD:-600}                        # seconds

LOG=${TESTSUITELOG:-$TMP/$(basename $0 .sh).log}
CLIENT=$SINGLECLIENT
NODES_TO_USE=${NODES_TO_USE:-$CLIENTS}
NUM_CLIENTS=$(get_node_count ${NODES_TO_USE//,/ })

rm -f $LOG

[ ! -x ${MDSRATE} ] && error "${MDSRATE} not built."

log "===== $0 ====== "

check_and_setup_lustre

mkdir -p $BASEDIR
chmod 0777 $BASEDIR
mdsrate_STRIPEPARAMS=${mdsrate_STRIPEPARAMS:-${fs_STRIPEPARAMS:-"-c 1"}}
setstripe_getstripe $BASEDIR $mdsrate_STRIPEPARAMS

IFree=$(($(mdsrate_inodes_available) - NUM_DIRS))
if [ $IFree -lt $((NUM_FILES * NUM_DIRS)) ]; then
    NUM_FILES=$((IFree / NUM_DIRS))
fi

generate_machine_file $NODES_TO_USE $MACHINEFILE || error "can not generate machinefile"

DIRfmt="${BASEDIR}/lookup-%d"

#
# Unlink the files created in the directories under $BASEDIR.
# FIXME: does it make sense to add the possibility to unlink dirfmt to mdsrate?
#
mdsrate_cleanup_all() {
	local i
	for i in $(seq 0 $NUM_DIRS); do
		mdsrate_cleanup $NUM_CLIENTS $MACHINEFILE $NUM_FILES \
				$BASEDIR/lookup-$i 'f%%d' --ignore
	done
}

if [ -n "$NOCREATE" ]; then
    echo "NOCREATE=$NOCREATE  => no file creation."
else
	mdsrate_cleanup_all

    log "===== $0 Test preparation: creating ${NUM_DIRS} dirs with ${NUM_FILES} files."

    COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --mknod
                        --ndirs ${NUM_DIRS} --dirfmt '${DIRfmt}'
                        --nfiles ${NUM_FILES} --filefmt 'f%%d'"

	echo "+" ${COMMAND}
	# For files creation we can use -np equal to NUM_DIRS
	# This is just a test preparation, does not matter how many threads we
	# use for files creation; we just should be aware that NUM_DIRS is less
	# than or equal to the number of threads np
	mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} -np ${NUM_DIRS} \
		${COMMAND} 2>&1

	# No lookup if error occurs on file creation, abort.
	if [ ${PIPESTATUS[0]} != 0 ]; then
		error_noexit "mdsrate file creation failed, aborting"
		mdsrate_cleanup_all
		exit 1
	fi
fi

COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --lookup --time ${TIME_PERIOD} ${SEED_OPTION}
        --ndirs ${NUM_DIRS} --dirfmt '${DIRfmt}'
        --nfiles ${NUM_FILES} --filefmt 'f%%d'"

# 1
if [ -n "$NOSINGLE" ]; then
    echo "NO Test for lookups on a single client."
else
	log "===== $0 ### 1 NODE LOOKUPS ###"
	echo "+" ${COMMAND}
	mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} -np 1 ${COMMAND} |
		tee ${LOG}

	if [ ${PIPESTATUS[0]} != 0 ]; then
		[ -f $LOG ] && sed -e "s/^/log: /" $LOG
		error_noexit "mdsrate lookup on single client failed, aborting"
		mdsrate_cleanup_all
		exit 1
	fi
fi

# 2
[ $NUM_CLIENTS -eq 1 ] && NOMULTI=yes
if [ -n "$NOMULTI" ]; then
    echo "NO test for lookups on multiple nodes."
else
	log "===== $0 ### ${NUM_CLIENTS} NODES LOOKUPS ###"
	echo "+" ${COMMAND}
	mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} -np ${NUM_CLIENTS} \
		${COMMAND} | tee ${LOG}

	if [ ${PIPESTATUS[0]} != 0 ]; then
		[ -f $LOG ] && sed -e "s/^/log: /" $LOG
		error_noexit "mdsrate lookup on multiple nodes failed, aborting"
		mdsrate_cleanup_all
		exit 1
	fi
fi

complete $SECONDS
mdsrate_cleanup_all
rmdir $BASEDIR || true
rm -f $MACHINEFILE
check_and_cleanup_lustre
#rm -f $LOG

exit 0


#!/bin/bash
#
# This test was used in a set of CMD3 tests (cmd3-5 test).

# Directory lookup retrieval rate single directory 10 million files
# 5900 random lookups/sec per client node 62,000 random lookups/sec aggregate
#
# In a dir containing 10 million non-striped files the mdsrate Test Program will
# perform lookups for 10 minutes. This test can be run from a single node for
# #1 and from all nodes for #2 aggregate test to measure lookup performance.

LUSTRE=${LUSTRE:-`dirname $0`/..}
. $LUSTRE/tests/test-framework.sh
init_test_env $@
. ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
assert_env CLIENTS MDSRATE SINGLECLIENT MPIRUN

MACHINEFILE=${MACHINEFILE:-$TMP/$(basename $0 .sh).machines}
# Do not use name [df][0-9]* to avoid cleanup by rm, bug 18045
BASEDIR=$MOUNT/mdsrate
TESTDIR=$BASEDIR/lookup

# Requirements
NUM_FILES=${NUM_FILES:-1000000}
TIME_PERIOD=${TIME_PERIOD:-600}                        # seconds

LOG=${TESTSUITELOG:-$TMP/$(basename $0 .sh).log}
CLIENT=$SINGLECLIENT
NODES_TO_USE=${NODES_TO_USE:-$CLIENTS}
NUM_CLIENTS=$(get_node_count ${NODES_TO_USE//,/ })

rm -f $LOG

[ ! -x ${MDSRATE} ] && error "${MDSRATE} not built."

log "===== $0 ====== "

check_and_setup_lustre

mkdir -p $BASEDIR
chmod 0777 $BASEDIR
mdsrate_STRIPEPARAMS=${mdsrate_STRIPEPARAMS:-${fs_STRIPEPARAMS:-"-c 1"}}
setstripe_getstripe $BASEDIR $mdsrate_STRIPEPARAMS

IFree=$(mdsrate_inodes_available)
if [ $IFree -lt $NUM_FILES ]; then
    NUM_FILES=$IFree
fi

generate_machine_file $NODES_TO_USE $MACHINEFILE || error "can not generate machinefile"

if [ -n "$NOCREATE" ]; then
    echo "NOCREATE=$NOCREATE  => no file creation."
else
    mdsrate_cleanup $NUM_CLIENTS $MACHINEFILE $NUM_FILES $TESTDIR 'f%%d' --ignore

    log "===== $0 Test preparation: creating ${NUM_FILES} files."

    NUM_CLIENTS=$(get_node_count ${NODES_TO_USE//,/ })
    NUM_THREADS=$((NUM_CLIENTS * MDSCOUNT))
    if [ $NUM_CLIENTS -gt 50 ]; then
        NUM_THREADS=$NUM_CLIENTS
    fi
    COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --mknod --dir ${TESTDIR}
                        --nfiles ${NUM_FILES} --filefmt 'f%%d'"
	echo "+" ${COMMAND}
	mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} -np ${NUM_THREADS} \
		${COMMAND} 2>&1

	# No lockup if error occurs on file creation, abort.
	if [ ${PIPESTATUS[0]} != 0 ]; then
		error_noexit "mdsrate file creation failed, aborting"
		mdsrate_cleanup $NUM_THREADS $MACHINEFILE $NUM_FILES \
				$TESTDIR 'f%%d' --ignore
		exit 1
	fi
fi

COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --lookup --time ${TIME_PERIOD} ${SEED_OPTION}
        --dir ${TESTDIR} --nfiles ${NUM_FILES} --filefmt 'f%%d'"

# 1
if [ -n "$NOSINGLE" ]; then
    echo "NO Test for lookups on a single client."
else
	log "===== $0 ### 1 NODE LOOKUPS ###"
	echo "+" ${COMMAND}
	mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} -np 1 ${COMMAND} |
		tee ${LOG}

	if [ ${PIPESTATUS[0]} != 0 ]; then
		[ -f $LOG ] && sed -e "s/^/log: /" $LOG
		error_noexit "mdsrate lookup on single client failed, aborting"
		mdsrate_cleanup $NUM_CLIENTS $MACHINEFILE $NUM_FILES \
				$TESTDIR 'f%%d' --ignore
		exit 1
	fi
fi

# 2
[ $NUM_CLIENTS -eq 1 ] && NOMULTI=yes
if [ -n "$NOMULTI" ]; then
    echo "NO test for lookups on multiple nodes."
else
	log "===== $0 ### ${NUM_CLIENTS} NODES LOOKUPS ###"
	echo "+" ${COMMAND}
	mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} -np ${NUM_CLIENTS} \
		${COMMAND} | tee ${LOG}

	if [ ${PIPESTATUS[0]} != 0 ]; then
		[ -f $LOG ] && sed -e "s/^/log: /" $LOG
		error_noexit "mdsrate lookup on multiple nodes failed, aborting"
		mdsrate_cleanup $NUM_CLIENTS $MACHINEFILE $NUM_FILES \
				$TESTDIR 'f%%d' --ignore
		exit 1
	fi
fi

complete $SECONDS
mdsrate_cleanup $NUM_CLIENTS $MACHINEFILE $NUM_FILES $TESTDIR 'f%%d'
rmdir $BASEDIR || true
rm -f $MACHINEFILE
check_and_cleanup_lustre
#rm -f $LOG

exit 0


#!/bin/bash
#
# This test was used in a set of CMD3 tests (cmd3-8 test).

# File attribute retrieval rate for large file creation
# 3300 ops/sec/OST for single node 28500 ops/sec/OST aggregate

# In a dir containing 10 million striped files, the mdsrate Test Program will
# perform directory ordered stat's (readdir) for 10 minutes. This test will be
# run from a single node for #1 and from all nodes for #2 aggregate test to
# measure stat performance.

LUSTRE=${LUSTRE:-`dirname $0`/..}
. $LUSTRE/tests/test-framework.sh
init_test_env $@
. ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}

assert_env CLIENTS MDSRATE SINGLECLIENT MPIRUN

MACHINEFILE=${MACHINEFILE:-$TMP/$(basename $0 .sh).machines}
# Do not use name [df][0-9]* to avoid cleanup by rm, bug 18045
BASEDIR=$MOUNT/mdsrate
TESTDIR=$MOUNT/stat

# Requirements
NUM_FILES=${NUM_FILES:-1000000}
TIME_PERIOD=${TIME_PERIOD:-600}                        # seconds

LOG=${TESTSUITELOG:-$TMP/$(basename $0 .sh).log}
CLIENT=$SINGLECLIENT
NODES_TO_USE=${NODES_TO_USE:-$CLIENTS}
NUM_CLIENTS=$(get_node_count ${NODES_TO_USE//,/ })

rm -f $LOG

[ ! -x ${MDSRATE} ] && error "${MDSRATE} not built."

log "===== $0 ====== "

check_and_setup_lustre

mkdir -p $BASEDIR
chmod 0777 $BASEDIR
mdsrate_STRIPEPARAMS=${mdsrate_STRIPEPARAMS:-${fs_STRIPEPARAMS:-"-c -1"}}
setstripe_getstripe $BASEDIR $mdsrate_STRIPEPARAMS

IFree=$(mdsrate_inodes_available)
if [ $IFree -lt $NUM_FILES ]; then
    NUM_FILES=$IFree
fi

generate_machine_file $NODES_TO_USE $MACHINEFILE || error "can not generate machinefile"

if [ -n "$NOCREATE" ]; then
    echo "NOCREATE=$NOCREATE  => no file creation."
else
    mdsrate_cleanup $NUM_CLIENTS $MACHINEFILE $NUM_FILES $TESTDIR 'f%%d' --ignore

    log "===== $0 Test preparation: creating ${NUM_FILES} files."

    COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --create --dir ${TESTDIR}
                        --nfiles ${NUM_FILES} --filefmt 'f%%d'"
    echo "+" ${COMMAND}

    NUM_CLIENTS=$(get_node_count ${NODES_TO_USE//,/ })
    NUM_THREADS=$((NUM_CLIENTS * MDSCOUNT))
    if [ $NUM_CLIENTS -gt 50 ]; then
        NUM_THREADS=$NUM_CLIENTS
    fi

	mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} -np ${NUM_THREADS} \
		${COMMAND} 2>&1

	if [ ${PIPESTATUS[0]} != 0 ]; then
		error_noexit "mdsrate file creation failed, aborting"
		mdsrate_cleanup $NUM_THREADS $MACHINEFILE $NUM_FILES \
				$TESTDIR 'f%%d' --ignore
		exit 1
	fi
fi

COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --stat --time ${TIME_PERIOD}
        --dir ${TESTDIR} --nfiles ${NUM_FILES} --filefmt 'f%%d'"

# 1
if [ -n "$NOSINGLE" ]; then
    echo "NO Test for stats on a single client."
else
    log "===== $0 ### 1 NODE STAT ###"
    echo "+" ${COMMAND}

	mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} -np 1 ${COMMAND} |
		tee ${LOG}

	if [ ${PIPESTATUS[0]} != 0 ]; then
		[ -f $LOG ] && sed -e "s/^/log: /" $LOG
		error_noexit "mdsrate stat on single client failed, aborting"
		mdsrate_cleanup $NUM_CLIENTS $MACHINEFILE $NUM_FILES \
				$TESTDIR 'f%%d' --ignore
		exit 1
	fi
fi

# 2
[ $NUM_CLIENTS -eq 1 ] && NOMULTI=yes
if [ -n "$NOMULTI" ]; then
    echo "NO test for stats on multiple nodes."
else
    log "===== $0 ### ${NUM_CLIENTS} NODES STAT ###"
    echo "+" ${COMMAND}

	mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} -np ${NUM_CLIENTS} \
		${COMMAND} | tee ${LOG}

	if [ ${PIPESTATUS[0]} != 0 ]; then
		[ -f $LOG ] && sed -e "s/^/log: /" $LOG
		error_noexit "mdsrate stat on multiple nodes failed, aborting"
		mdsrate_cleanup $NUM_CLIENTS $MACHINEFILE $NUM_FILES \
				$TESTDIR 'f%%d' --ignore
		exit 1
	fi
fi

complete $SECONDS
mdsrate_cleanup $NUM_CLIENTS $MACHINEFILE $NUM_FILES $TESTDIR 'f%%d'
rmdir $BASEDIR || true
rm -f $MACHINEFILE
check_and_cleanup_lustre
#rm -f $LOG

exit 0


#!/bin/bash
#
# This test was used in a set of CMD3 tests (cmd3-7 test).

# File attribute retrieval rate for small file creation
# 3200 ops/sec for single node 29,000 ops/sec aggregate

# In a dir containing 10 million non-striped files, the mdsrate Test Program
# will perform directory ordered stat's (readdir) for 10 minutes. This test
# will be run from a single node for #1 and from all nodes for #2
# aggregate test to measure stat performance.

LUSTRE=${LUSTRE:-`dirname $0`/..}
. $LUSTRE/tests/test-framework.sh
init_test_env $@
. ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}

assert_env CLIENTS MDSRATE SINGLECLIENT MPIRUN

MACHINEFILE=${MACHINEFILE:-$TMP/$(basename $0 .sh).machines}
# Do not use name [df][0-9]* to avoid cleanup by rm, bug 18045
BASEDIR=$MOUNT/mdsrate
TESTDIR=$BASEDIR/stat

# Requirements
NUM_FILES=${NUM_FILES:-1000000}
TIME_PERIOD=${TIME_PERIOD:-600}                        # seconds

LOG=${TESTSUITELOG:-$TMP/$(basename $0 .sh).log}
CLIENT=$SINGLECLIENT
NODES_TO_USE=${NODES_TO_USE:-$CLIENTS}
NUM_CLIENTS=$(get_node_count ${NODES_TO_USE//,/ })

rm -f $LOG

[ ! -x ${MDSRATE} ] && error "${MDSRATE} not built."

log "===== $0 ====== "

check_and_setup_lustre

mkdir -p $BASEDIR
chmod 0777 $BASEDIR
mdsrate_STRIPEPARAMS=${mdsrate_STRIPEPARAMS:-${fs_STRIPEPARAMS:-"-i 0 -c 1"}}
setstripe_getstripe $BASEDIR $mdsrate_STRIPEPARAMS

IFree=$(mdsrate_inodes_available)
if [ $IFree -lt $NUM_FILES ]; then
    NUM_FILES=$IFree
fi

generate_machine_file $NODES_TO_USE $MACHINEFILE || error "can not generate machinefile"

if [ -n "$NOCREATE" ]; then
    echo "NOCREATE=$NOCREATE  => no file creation."
else
    mdsrate_cleanup $NUM_CLIENTS $MACHINEFILE $NUM_FILES $TESTDIR 'f%%d' --ignore

    log "===== $0 Test preparation: creating ${NUM_FILES} files."

    COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --mknod --dir ${TESTDIR}
                        --nfiles ${NUM_FILES} --filefmt 'f%%d'"
    echo "+" ${COMMAND}

    NUM_CLIENTS=$(get_node_count ${NODES_TO_USE//,/ })
    NUM_THREADS=$((NUM_CLIENTS * MDSCOUNT))
    if [ $NUM_CLIENTS -gt 50 ]; then
        NUM_THREADS=$NUM_CLIENTS
    fi

	mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} -np ${NUM_THREADS} \
		${COMMAND} 2>&1

	if [ ${PIPESTATUS[0]} != 0 ]; then
		error_noexit "mdsrate file creation failed, aborting"
		mdsrate_cleanup $NUM_THREADS $MACHINEFILE $NUM_FILES \
				$TESTDIR 'f%%d' --ignore
		exit 1
	fi
fi

COMMAND="${MDSRATE} ${MDSRATE_DEBUG} --stat --time ${TIME_PERIOD}
        --dir ${TESTDIR} --nfiles ${NUM_FILES} --filefmt 'f%%d'"

# 1
if [ -n "$NOSINGLE" ]; then
    echo "NO Test for stats on a single client."
else
    log "===== $0 ### 1 NODE STAT ###"
    echo "+" ${COMMAND}

	mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} -np 1 ${COMMAND} |
		tee ${LOG}

	if [ ${PIPESTATUS[0]} != 0 ]; then
		[ -f $LOG ] && sed -e "s/^/log: /" $LOG
		error_noexit "mdsrate stat on single client failed, aborting"
		mdsrate_cleanup $NUM_CLIENTS $MACHINEFILE $NUM_FILES \
				$TESTDIR 'f%%d' --ignore
		exit 1
	fi
fi

# 2
[ $NUM_CLIENTS -eq 1 ] && NOMULTI=yes
if [ -n "$NOMULTI" ]; then
    echo "NO test for stats on multiple nodes."
else
    log "===== $0 ### ${NUM_CLIENTS} NODES STAT ###"
    echo "+" ${COMMAND}

	mpi_run ${MACHINEFILE_OPTION} ${MACHINEFILE} -np ${NUM_CLIENTS} \
		${COMMAND} | tee ${LOG}

	if [ ${PIPESTATUS[0]} != 0 ]; then
		[ -f $LOG ] && sed -e "s/^/log: /" $LOG
		error_noexit "mdsrate stat on multiple nodes failed, aborting"
		mdsrate_cleanup $NUM_CLIENTS $MACHINEFILE $NUM_FILES \
				$TESTDIR 'f%%d' --ignore
		exit 1
	fi
fi

complete $SECONDS
mdsrate_cleanup $NUM_CLIENTS $MACHINEFILE $NUM_FILES $TESTDIR 'f%%d'
rmdir $BASEDIR || true
rm -f $MACHINEFILE
check_and_cleanup_lustre
#rm -f $LOG

exit 0


#!/bin/bash

# A Metadata Update Test tests that
# metadata updates are properly completed when
# multiple clients create/delete files and modify the attributes of files.

set -e

ONLY=${ONLY:-"$*"}

#Bug number for excepting test
ALWAYS_EXCEPT="$METADATA_UPDATES_EXCEPT"
# UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT LIST

[ "$SLOW" = "no" ] && EXCEPT_SLOW=""

LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
. $LUSTRE/tests/test-framework.sh
init_test_env $@
. ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
init_logging

TRACE=${TRACE:-"+x"}

TESTDIR=${TESTDIR:-$DIR/d0.$(basename $0 .sh)}

NODES_TO_USE=${NODES_TO_USE:-$CLIENTS}

[ -z $CLIENTS ] && NODES_TO_USE=$(hostname)

# hostname could differ from a network interface
# configured for NODES_TO_USE, bug 23961
# the test dir on each host is created based on `hostname` of this host
HOSTS=$(comma_list $(do_nodes $NODES_TO_USE "echo \\\$(hostname)"))

FILE=testfile
FILE_SIZE=1024
CURRENT_MODE=0644
NEW_MODE=0222
NEW_ATIME="2001-01-01 GMT"
NEW_MTIME="2005-05-05 GMT"

test_USER=$(id -u -n)
test_GROUP=$(id -g -n)

SUMFILE=$TESTDIR/mdsum

NUM_FILES=1000

log "===== $0 ====== "

check_and_setup_lustre
build_test_filter

cleanup_prepare () {

	do_nodes $NODES_TO_USE "set $TRACE;
	DIR=$TESTDIR/\\\$(hostname);
	TESTFILE=\\\$DIR/$FILE;
	rm -f \\\$TESTFILE;
	rm -f $SUMFILE;
	rmdir \\\$DIR 2>/dev/null;
	mkdir -p \\\$DIR" || return ${PIPESTATUS[0]}
	return 0;
}

do_mknod () {
	log "Creating file(s) by mknod (2) ... "

	do_nodes $NODES_TO_USE "set $TRACE;
	TESTFILE=$TESTDIR/\\\$(hostname)/$FILE;
	mcreate \\\$TESTFILE; " || return ${PIPESTATUS[0]}
	return 0
}

do_write () {
	do_nodes $NODES_TO_USE "set $TRACE;
	TESTFILE=$TESTDIR/\\\$(hostname)/$FILE;
	dd if=/dev/zero of=\\\$TESTFILE bs=$FILE_SIZE count=1 2>/dev/null ||
		exit 54;
	echo \\\$(hostname) | dd of=\\\$TESTFILE conv=notrunc 2>/dev/null ||
		exit 55;
	md5sum \\\$TESTFILE >> $SUMFILE; " || return ${PIPESTATUS[0]}
	return 0
}

do_check_data () {
	log "Checking file(s) data ... md5sum : "
	cat $SUMFILE

	do_nodesv $NODES_TO_USE "md5sum --check $SUMFILE" ||
		return ${PIPESTATUS[0]}
	return 0
}

do_truncate () {
	log "Truncating file(s) ... "

	do_nodes $NODES_TO_USE "set $TRACE;
	TESTFILE=$TESTDIR/\\\$(hostname)/$FILE;
	$TRUNCATE \\\$TESTFILE 0" || return ${PIPESTATUS[0]}

	FILE_SIZE=0
	return 0
}

# check st_uid, st_gid, st_size, st_mode
get_stat () {
	local attr="$test_USER $test_GROUP $FILE_SIZE $CURRENT_MODE"

	log "Checking file(s) attributes ... "

	do_nodesv $NODES_TO_USE "set $TRACE;
for HOST in ${HOSTS//,/ } ; do
    TESTFILE=$TESTDIR/\\\$HOST/$FILE;
    tmp=\\\$(stat -c \\\"%U %G %s 0%a\\\" \\\$TESTFILE);
    echo \\\"\\\$TESTFILE [ uid gid size mode ] expected : $attr ;  got : \\\$tmp \\\";
    if [ x\\\"\\\$tmp\\\" != x\\\"$attr\\\" ] ; then
        echo \\\"Wrong file attributes\\\";
        exit 56;
    fi;
done " || return ${PIPESTATUS[0]}
	return 0
}

do_chmod () {
	log "Performing chmod 0$NEW_MODE ..."

	do_nodes $NODES_TO_USE "set $TRACE;
	TESTFILE=$TESTDIR/\\\$(hostname)/$FILE;
	chmod $NEW_MODE \\\$TESTFILE" || return ${PIPESTATUS[0]}

	CURRENT_MODE=$NEW_MODE
	return 0
}

do_change_timestamps () {
	log "Changing atime, mtime ..."

	do_nodes $NODES_TO_USE " set $TRACE;
	TESTFILE=$TESTDIR/\\\$(hostname)/$FILE;
	touch -c --date=\\\"$NEW_ATIME\\\" -a \\\$TESTFILE;
	touch -c --date=\\\"$NEW_MTIME\\\" -m \\\$TESTFILE " ||
		return ${PIPESTATUS[0]}
	return 0
}

# check st_atime, st_mtime
do_check_timestamps () {
	local atime=$(date --date="$NEW_ATIME" +%s)
	local mtime=$(date --date="$NEW_MTIME" +%s)

	local times="$atime $mtime"

	log "Checking atime, mtime ... "

	do_nodesv $NODES_TO_USE "set $TRACE;
for HOST in ${HOSTS//,/ } ; do
    TESTFILE=$TESTDIR/\\\$HOST/$FILE;
    tmp=\\\$(stat -c \\\"%X %Y\\\" \\\$TESTFILE);
    if [ x\\\"\\\$tmp\\\" != x\\\"$times\\\" ] ; then
       echo \\\"\\\$TESTFILE [ atime mtime ] expected : $times ;  got : \\\$tmp \\\";
       RC=57;
    fi;
done;
exit \\\$RC" || return ${PIPESTATUS[0]}
	return 0
}

do_fill_dir () {
	log "Filling up directories ... files : f1 ... f$NUM_FILES) ... "

	do_nodes $NODES_TO_USE "set $TRACE;
	TESTFILE=$TESTDIR/\\\$(hostname)/$FILE;
	rm -f \\\$TESTFILE;
	DIR=$TESTDIR/\\\$(hostname);
	for i in \\\$(seq $NUM_FILES) ; do
		touch \\\$DIR/f\\\$i;
	done " || return ${PIPESTATUS[0]}
	return 0
}

check_dir_contents () {
	local num_files=${1:-1}

	log "Directory contents should exist: f$num_files ... f$NUM_FILES)"
	do_nodes $NODES_TO_USE "set $TRACE;
for HOST in ${HOSTS//,/ } ; do
    DIR=$TESTDIR/\\\$HOST;
    for i in \\\$(seq $NUM_FILES -1 $num_files) ; do
        if ! [ -f \\\$DIR/f\\\$i ] ; then
            echo \\\"ERROR: file \\\$DIR/f\\\$i should exist\\\";
            RC=1;
        fi;
    done;
    for i in \\\$(seq $(($num_files - 1 ))) ; do
        if [ -f \\\$DIR/f\\\$i ] ; then
            echo \\\"ERROR: deleted file \\\$DIR/f\\\$i exists\\\";
            RC=1;
        fi;
    done;
done;
exit \\\$RC " || return ${PIPESTATUS[0]}
	return 0
}

do_partial_delete () {
	local num_files=$1

	log "Deleting files ... f1 ... f$num_files ... "
	do_nodes $NODES_TO_USE "set $TRACE;
DIR=$TESTDIR/\\\$(hostname);
for i in \\\$(seq $num_files) ; do
    if ! rm -f \\\$DIR/f\\\$i ; then
        exit 1;
    fi;
done " || return ${PIPESTATUS[0]}
	return 0
}

chmod 0777 $MOUNT   || exit 1
mkdir -p $TESTDIR   || exit 1
chmod 0777 $TESTDIR || exit 1

cleanup_prepare     || error_exit "cleanup failed"

test_1(){
	log "Create files (mknod (2)) and write data"
	do_mknod || error "mknod failed"
	do_write || error "write data failed"
	log "Check data and file attributes."
	do_check_data || error "md5sum verification failed"
	get_stat || error "attributes check failed"

	# file attributes modification
	log "File attributes modification."
	do_chmod || error "chmod failed"
	get_stat || error "wrong attributes after chmod"

	do_change_timestamps || error "timestamps change failed"
	do_check_timestamps || error "wrong timestamps"

	# truncate file(s) to 0 size, check new file size
	log "Truncate file(s) to 0 size, check new file size."
	do_truncate || error "truncate failed"
	get_stat || error "wrong attributes after truncate"
}
run_test 1 "create files (mknod), write and check data, truncate files"

test_2() {
	do_fill_dir || error "dir creation failed"
	check_dir_contents || error "dir contents check failed"

	do_partial_delete $(($NUM_FILES / 2)) || error "delete failed"
	check_dir_contents $(($NUM_FILES / 2 + 1)) ||
		error "dir contents check after delete failed"
}
run_test 2 "directory content create, check, delete files , check"

test_3() {
	WRITE_DISJOINT=${WRITE_DISJOINT:-$(which write_disjoint 2> /dev/null)} || true
	disjoint_file=$TESTDIR/$tfile
	machine_file=${MACHINEFILE:-$TMP/$(basename $0 .sh).machines}
	numloops=1000

	[ ! -f "$WRITE_DISJOINT" ] && skip_env "write_disjoint not found"

	set $TRACE
	generate_machine_file $NODES_TO_USE $machine_file
	mpi_run ${MACHINEFILE_OPTION} $machine_file \
		-np $(get_node_count ${NODES_TO_USE//,/ }) \
		$WRITE_DISJOINT -f $disjoint_file -n $numloops ||
			error "mpi_run failed"

	rm -f $machine_file
}
run_test 3 "write_disjoint test"

complete $SECONDS
rm -rf $TESTDIR
check_and_cleanup_lustre
exit_status


#!/bin/bash
#
# Tests for multiple mount protection (MMP) feature.
#
# Run select tests by setting ONLY, or as arguments to the script.
# Skip specific tests by setting EXCEPT.
#
# e.g. ONLY="5 6" or ONLY="`seq 8 11`" or EXCEPT="7"
set -e

ONLY=${ONLY:-"$*"}

LUSTRE=${LUSTRE:-$(dirname $0)/..}
. $LUSTRE/tests/test-framework.sh
init_test_env $@
init_logging

# bug number for skipped test:
ALWAYS_EXCEPT=$MMP_EXCEPT
# UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!

build_test_filter

remote_mds_nodsh && skip "remote MDS with nodsh"
remote_ost_nodsh && skip "remote OST with nodsh"

# unmount and cleanup the Lustre filesystem
MMP_RESTORE_MOUNT=false
if is_mounted $MOUNT || is_mounted $MOUNT2; then
	cleanupall
	MMP_RESTORE_MOUNT=true
fi

SAVED_FAIL_ON_ERROR=$FAIL_ON_ERROR
FAIL_ON_ERROR=false

# Get the failover facet.
get_failover_facet() {
    local facet=$1
    local failover_facet=${facet}failover

    local host=$(facet_host $facet)
    local failover_host=$(facet_host $failover_facet)

    [ -z "$failover_host" -o "$host" = "$failover_host" ] && \
        failover_facet=$facet

    echo $failover_facet
}

# Initiate the variables for Lustre servers and targets.
init_vars() {
    MMP_MDS=${MMP_MDS:-$SINGLEMDS}
    MMP_MDS_FAILOVER=$(get_failover_facet $MMP_MDS)

    local mds_num=$(echo $MMP_MDS | tr -d "mds")
    MMP_MDSDEV=$(mdsdevname $mds_num)

    MMP_OSS=${MMP_OSS:-ost1}
    MMP_OSS_FAILOVER=$(get_failover_facet $MMP_OSS)

    local oss_num=$(echo $MMP_OSS | tr -d "ost")
    MMP_OSTDEV=$(ostdevname $oss_num)
}

# Stop the MDS and OSS services on the primary or failover servers.
stop_services() {
    local flavor=$1
    shift
    local opts="$@"
    local mds_facet
    local oss_facet

    if [ "$flavor" = "failover" ]; then
        mds_facet=$MMP_MDS_FAILOVER
        oss_facet=$MMP_OSS_FAILOVER
    else
        mds_facet=$MMP_MDS
        oss_facet=$MMP_OSS
    fi

    stop $mds_facet $opts || return ${PIPESTATUS[0]}
    stop $oss_facet $opts || return ${PIPESTATUS[0]}
}

# Enable the MMP feature.
enable_mmp() {
    local facet=$1
    local device=$2

    do_facet $facet "$TUNE2FS -O mmp $device"
    return ${PIPESTATUS[0]}
}

# Disable the MMP feature.
disable_mmp() {
    local facet=$1
    local device=$2

    do_facet $facet "$TUNE2FS -O ^mmp $device"
    return ${PIPESTATUS[0]}
}

# Set the MMP block to 'fsck' state
mark_mmp_block() {
    local facet=$1
    local device=$2

    do_facet $facet "$LUSTRE/tests/mmp_mark.sh $device"
    return ${PIPESTATUS[0]}
}

# Reset the MMP block (if any) back to the clean state.
reset_mmp_block() {
    local facet=$1
    local device=$2

    do_facet $facet "$TUNE2FS -f -E clear-mmp $device"
    return ${PIPESTATUS[0]}
}

# Check whether the MMP feature is enabled or not.
mmp_is_enabled() {
    local facet=$1
    local device=$2

    do_facet $facet "$DUMPE2FS -h $device | grep mmp"
    return ${PIPESTATUS[0]}
}

# Get MMP update interval (in seconds) from the Lustre server target.
get_mmp_update_interval() {
	local facet=$1
	local device=$2
	local interval

	interval=$(do_facet $facet \
		   "$DEBUGFS -c -R dump_mmp $device 2>$TMP/mmp.debugfs.msg" |
		   awk 'tolower($0) ~ /update.interval/ { print $NF }')
	[ -z "$interval" ] && interval=5 &&
		do_facet $facet cat $TMP/mmp.debugfs.msg &&
		echo "$facet:$device: assume update interval=$interval" 1>&2 ||
		echo "$facet:$device: got actual update interval=$interval" 1>&2

	echo $interval
}

# Get MMP check interval (in seconds) from the Lustre server target.
get_mmp_check_interval() {
	local facet=$1
	local device=$2
	local interval

	interval=$(do_facet $facet \
		   "$DEBUGFS -c -R dump_mmp $device 2>$TMP/mmp.debugfs.msg" |
		   awk 'tolower($0) ~ /check.interval/ { print $NF }')
	[ -z "$interval" ] && interval=5 &&
		do_facet $facet cat $TMP/mmp.debugfs.msg &&
		echo "$facet:$device: assume check interval=$interval" 1>&2 ||
		echo "$facet:$device: got actual check interval=$interval" 1>&2

	echo $interval
}

# Adjust the MMP update interval (in seconds) on the Lustre server target.
# Specifying an interval of 0 means to use the default interval.
set_mmp_update_interval() {
	local facet=$1
	local device=$2
	local interval=${3:-0}

	do_facet $facet "$TUNE2FS -E mmp_update_interval=$interval $device"
	return ${PIPESTATUS[0]}
}

I_ENABLED_MDS=0
I_ENABLED_OSS=0

# Enable the MMP feature on the Lustre server targets.
mmp_init() {
	init_vars

	if [ $(facet_fstype $MMP_MDS) != ldiskfs ]; then
		skip_env "ldiskfs only test"
	fi

	if [ $(facet_fstype $MMP_OSS) != ldiskfs ]; then
		skip_env "ldiskfs only test"
	fi

	mmp_is_enabled $MMP_MDS $MMP_MDSDEV ||
	{
		log "MMP is not enabled on MDS, enabling it manually..."
		enable_mmp $MMP_MDS $MMP_MDSDEV ||
			error "failed to enable MMP on $MMP_MDSDEV on $MMP_MDS"
		I_ENABLED_MDS=1
	}

	mmp_is_enabled $MMP_OSS $MMP_OSTDEV ||
	{
		log "MMP is not enabled on OSS, enabling it manually..."
		enable_mmp $MMP_OSS $MMP_OSTDEV ||
			error "failed to enable MMP on $MMP_OSTDEV on $MMP_OSS"
		I_ENABLED_OSS=1
	}

	# check whether the MMP feature is enabled or not
	mmp_is_enabled $MMP_MDS $MMP_MDSDEV ||
		error "MMP was not enabled on $MMP_MDSDEV on $MMP_MDS"

	mmp_is_enabled $MMP_OSS $MMP_OSTDEV ||
		error "MMP was not enabled on $MMP_OSTDEV on $MMP_OSS"
}

# Disable the MMP feature on the Lustre server targets
mmp_fini() {

	if [ $I_ENABLED_MDS -eq 1 ]; then
		log "Disabling MMP on $MMP_MDSDEV on $MMP_MDS manually..."
		disable_mmp $MMP_MDS $MMP_MDSDEV ||
			error "failed to disable MMP on $MMP_MDSDEV on $MMP_MDS"
		mmp_is_enabled $MMP_MDS $MMP_MDSDEV &&
			error "MMP was not disabled on $MMP_MDSDEV on $MMP_MDS"
	fi

	if [ $I_ENABLED_OSS -eq 1 ]; then
		log "Disabling MMP on $MMP_OSTDEV on $MMP_OSS manually..."
		disable_mmp $MMP_OSS $MMP_OSTDEV ||
			error "failed to disable MMP on $MMP_OSTDEV on $MMP_OSS"
		mmp_is_enabled $MMP_OSS $MMP_OSTDEV &&
			error "MMP was not disabled on $MMP_OSTDEV on $MMP_OSS"
	fi

	return 0
}

# Mount the shared target on the failover server after some interval it's
# mounted on the primary server.
mount_after_interval_sub() {
    local interval=$1
    shift
    local device=$1
    shift
    local facet=$1
    shift
    local opts="$@"
    local failover_facet=$(get_failover_facet $facet)

    local mount_pid
    local first_mount_rc=0
    local second_mount_rc=0

    log "Mounting $device on $facet..."
    start $facet $device $opts &
    mount_pid=$!

    if [ $interval -ne 0 ]; then
        log "sleep $interval..."
        sleep $interval
    fi

    log "Mounting $device on $failover_facet..."
    start $failover_facet $device $opts
    second_mount_rc=${PIPESTATUS[0]}

    wait $mount_pid
    first_mount_rc=${PIPESTATUS[0]}

    if [ $second_mount_rc -eq 0 -a $first_mount_rc -eq 0 ]; then
        error_noexit "one mount delayed by mmp interval $interval should fail"
        stop $facet || return ${PIPESTATUS[0]}
        [ "$failover_facet" != "$facet" ] && stop $failover_facet || \
            return ${PIPESTATUS[0]}
        return 1
    elif [ $second_mount_rc -ne 0 -a $first_mount_rc -ne 0 ]; then
	error_noexit "mount failure on failover pair $facet,$failover_facet"
        return $first_mount_rc
    fi

    return 0
}

mount_after_interval() {
    local mdt_interval=$1
    local ost_interval=$2
    local rc=0

    mount_after_interval_sub $mdt_interval $MMP_MDSDEV $MMP_MDS \
        $MDS_MOUNT_OPTS || return ${PIPESTATUS[0]}

    echo
    mount_after_interval_sub $ost_interval $MMP_OSTDEV $MMP_OSS $OST_MOUNT_OPTS
    rc=${PIPESTATUS[0]}
    if [ $rc -ne 0 ]; then
        stop $MMP_MDS
        return $rc
    fi

    return 0
}

# Mount the shared target on the failover server
# during unmounting it on the primary server.
mount_during_unmount() {
	local device=$1
	shift
	local facet=$1
	shift
	local mnt_opts="$@"
	local failover_facet=$(get_failover_facet $facet)

	local unmount_pid
	local unmount_rc=0
	local mount_rc=0

	log "Mounting $device on $facet..."
	start $facet $device $mnt_opts || return ${PIPESTATUS[0]}

	log "Unmounting $device on $facet..."
	stop $facet &
	unmount_pid=$!

	log "Mounting $device on $failover_facet..."
	start $failover_facet $device $mnt_opts
	mount_rc=${PIPESTATUS[0]}

	# check whether the first filesystem is still mounted
	local mntpt=$(facet_mntpt $facet)
	local mounted=$(do_facet $facet "grep -w $mntpt /proc/mounts")

	wait $unmount_pid
	unmount_rc=${PIPESTATUS[0]}

	if [ $mount_rc -eq 0 ]; then
		stop $failover_facet || return ${PIPESTATUS[0]}

		if [ -n "$mounted" ]; then
			error_noexit "mount during unmount of first filesystem worked"
			return 1
		fi
	fi

	if [ $unmount_rc -ne 0 ]; then
		error_noexit "unmount the $device on $facet should succeed"
		return $unmount_rc
	fi

	return 0
}

# Mount the shared target on the failover server
# after clean unmounting it on the primary server.
mount_after_unmount() {
    local device=$1
    shift
    local facet=$1
    shift
    local mnt_opts="$@"
    local failover_facet=$(get_failover_facet $facet)

    log "Mounting $device on $facet..."
    start $facet $device $mnt_opts || return ${PIPESTATUS[0]}

    log "Unmounting $device on $facet..."
    stop $facet || return ${PIPESTATUS[0]}

    log "Mounting $device on $failover_facet..."
    start $failover_facet $device $mnt_opts || return ${PIPESTATUS[0]}

    return 0
}

# Mount the shared target on the failover server after rebooting
# the primary server.
mount_after_reboot() {
    local device=$1
    shift
    local facet=$1
    shift
    local mnt_opts="$@"
    local failover_facet=$(get_failover_facet $facet)
    local rc=0

    log "Mounting $device on $facet..."
    start $facet $device $mnt_opts || return ${PIPESTATUS[0]}

    if [ "$FAILURE_MODE" = "HARD" ]; then
        shutdown_facet $facet
        reboot_facet $facet
        wait_for_facet $facet
    else
        replay_barrier_nodf $facet
    fi

    log "Mounting $device on $failover_facet..."
    start $failover_facet $device $mnt_opts
    rc=${PIPESTATUS[0]}
    if [ $rc -ne 0 ]; then
        error_noexit "mount $device on $failover_facet should succeed"
        stop $facet || return ${PIPESTATUS[0]}
        return $rc
    fi

    return 0
}

# Run e2fsck on the Lustre server target.
run_e2fsck() {
	local facet=$1
	shift
	local device=$1
	shift
	local opts="$@"

	echo "Running e2fsck on the device $device on $facet..."
	do_facet $facet "$E2FSCK $opts $device"
	return ${PIPESTATUS[0]}
}

# Check whether there are failover pairs for MDS and OSS servers.
check_failover_pair() {
	[ "$MMP_MDS" = "$MMP_MDS_FAILOVER" -o "$MMP_OSS" = "$MMP_OSS_FAILOVER" ] &&
		skip_env "failover pair is needed"
	return 0
}

mmp_init

# Test 1 - two mounts at the same time.
test_1() {
	check_failover_pair

	mount_after_interval 0 0 || return ${PIPESTATUS[0]}
	stop_services primary || return ${PIPESTATUS[0]}
	stop_services failover || return ${PIPESTATUS[0]}
}
run_test 1 "two mounts at the same time"

# Test 2 - one mount delayed by mmp update interval.
test_2() {
	check_failover_pair

	local mdt_interval=$(get_mmp_update_interval $MMP_MDS $MMP_MDSDEV)
	local ost_interval=$(get_mmp_update_interval $MMP_OSS $MMP_OSTDEV)

	mount_after_interval $mdt_interval $ost_interval ||
		return ${PIPESTATUS[0]}
	stop_services primary || return ${PIPESTATUS[0]}
}
run_test 2 "one mount delayed by mmp update interval"

# Test 3 - one mount delayed by 2x mmp check interval.
test_3() {
	check_failover_pair

	local mdt_interval=$(get_mmp_check_interval $MMP_MDS $MMP_MDSDEV)
	local ost_interval=$(get_mmp_check_interval $MMP_OSS $MMP_OSTDEV)

	mdt_interval=$((2 * $mdt_interval + 1))
	ost_interval=$((2 * $ost_interval + 1))

	mount_after_interval $mdt_interval $ost_interval ||
		return ${PIPESTATUS[0]}
	stop_services primary || return ${PIPESTATUS[0]}
}
run_test 3 "one mount delayed by 2x mmp check interval"

# Test 4 - one mount delayed by > 2x mmp check interval.
test_4() {
	check_failover_pair

	local mdt_interval=$(get_mmp_check_interval $MMP_MDS $MMP_MDSDEV)
	local ost_interval=$(get_mmp_check_interval $MMP_OSS $MMP_OSTDEV)

	mdt_interval=$((4 * $mdt_interval))
	ost_interval=$((4 * $ost_interval))

	mount_after_interval $mdt_interval $ost_interval ||
		return ${PIPESTATUS[0]}
	stop_services primary || return ${PIPESTATUS[0]}
}
run_test 4 "one mount delayed by > 2x mmp check interval"

# Test 5 - mount during unmount of the first filesystem.
test_5() {
	local rc=0
	check_failover_pair

	mount_during_unmount $MMP_MDSDEV $MMP_MDS $MDS_MOUNT_OPTS ||
		return ${PIPESTATUS[0]}

	echo
	start $MMP_MDS $MMP_MDSDEV $MDS_MOUNT_OPTS || return ${PIPESTATUS[0]}
	mount_during_unmount $MMP_OSTDEV $MMP_OSS $OST_MOUNT_OPTS
	rc=${PIPESTATUS[0]}
	stop $MMP_MDS || return ${PIPESTATUS[0]}
	return $rc
}
run_test 5 "mount during unmount of the first filesystem"

# Test 6 - mount after clean unmount.
test_6() {
	local rc=0
	check_failover_pair

	mount_after_unmount $MMP_MDSDEV $MMP_MDS $MDS_MOUNT_OPTS ||
		return ${PIPESTATUS[0]}

	echo
	mount_after_unmount $MMP_OSTDEV $MMP_OSS $OST_MOUNT_OPTS
	rc=${PIPESTATUS[0]}
	if [ $rc -ne 0 ]; then
		stop $MMP_MDS_FAILOVER || return ${PIPESTATUS[0]}
		return $rc
	fi

	stop_services failover || return ${PIPESTATUS[0]}
}
run_test 6 "mount after clean unmount"

# Test 7 - mount after reboot.
test_7() {
	local rc=0
	check_failover_pair

	mount_after_reboot $MMP_MDSDEV $MMP_MDS $MDS_MOUNT_OPTS ||
		return ${PIPESTATUS[0]}

	echo
	mount_after_reboot $MMP_OSTDEV $MMP_OSS $OST_MOUNT_OPTS
	rc=${PIPESTATUS[0]}
	if [ $rc -ne 0 ]; then
		stop $MMP_MDS || return ${PIPESTATUS[0]}
		stop $MMP_MDS_FAILOVER || return ${PIPESTATUS[0]}
		return $rc
	fi

	stop_services failover || return ${PIPESTATUS[0]}
	stop_services primary || return ${PIPESTATUS[0]}
}
run_test 7 "mount after reboot"

# Test 8 - mount during e2fsck (should never succeed).
test_8() {
	local e2fsck_pid
	local saved_interval
	local new_interval

	# After writing a new sequence number into the MMP block, e2fsck will
	# sleep at least (2 * new_interval + 1) seconds before it goes into
	# e2fsck passes.
	new_interval=30

	# MDT
	saved_interval=$(get_mmp_update_interval $MMP_MDS $MMP_MDSDEV)
	set_mmp_update_interval $MMP_MDS $MMP_MDSDEV $new_interval

	run_e2fsck $MMP_MDS $MMP_MDSDEV "-fy" &
	e2fsck_pid=$!
	sleep 5

	if start $MMP_MDS_FAILOVER $MMP_MDSDEV $MDS_MOUNT_OPTS; then
		error_noexit \
			"mount $MMP_MDSDEV on $MMP_MDS_FAILOVER should fail"
		stop $MMP_MDS_FAILOVER || return ${PIPESTATUS[0]}
		set_mmp_update_interval $MMP_MDS $MMP_MDSDEV $saved_interval
		return 1
	fi

	wait $e2fsck_pid
	set_mmp_update_interval $MMP_MDS $MMP_MDSDEV $saved_interval

	# OST
	echo
	saved_interval=$(get_mmp_update_interval $MMP_OSS $MMP_OSTDEV)
	set_mmp_update_interval $MMP_OSS $MMP_OSTDEV $new_interval

	run_e2fsck $MMP_OSS $MMP_OSTDEV "-fy" &
	e2fsck_pid=$!
	sleep 5

	if start $MMP_OSS_FAILOVER $MMP_OSTDEV $OST_MOUNT_OPTS; then
		error_noexit \
			"mount $MMP_OSTDEV on $MMP_OSS_FAILOVER should fail"
		stop $MMP_OSS_FAILOVER || return ${PIPESTATUS[0]}
		set_mmp_update_interval $MMP_OSS $MMP_OSTDEV $saved_interval
		return 2
	fi

	wait $e2fsck_pid
	set_mmp_update_interval $MMP_OSS $MMP_OSTDEV $saved_interval
	return 0
}
run_test 8 "mount during e2fsck"

# Test 9 - mount after aborted e2fsck (should never succeed).
test_9() {
    start $MMP_MDS $MMP_MDSDEV $MDS_MOUNT_OPTS || return ${PIPESTATUS[0]}
    if ! start $MMP_OSS $MMP_OSTDEV $OST_MOUNT_OPTS; then
        local rc=${PIPESTATUS[0]}
        stop $MMP_MDS || return ${PIPESTATUS[0]}
        return $rc
    fi
    stop_services primary || return ${PIPESTATUS[0]}

    mark_mmp_block $MMP_MDS $MMP_MDSDEV || return ${PIPESTATUS[0]}

    log "Mounting $MMP_MDSDEV on $MMP_MDS..."
    if start $MMP_MDS $MMP_MDSDEV $MDS_MOUNT_OPTS; then
        error_noexit "mount $MMP_MDSDEV on $MMP_MDS should fail"
        stop $MMP_MDS || return ${PIPESTATUS[0]}
        return 1
    fi

    reset_mmp_block $MMP_MDS $MMP_MDSDEV || return ${PIPESTATUS[0]}

    mark_mmp_block $MMP_OSS $MMP_OSTDEV || return ${PIPESTATUS[0]}

    log "Mounting $MMP_OSTDEV on $MMP_OSS..."
    if start $MMP_OSS $MMP_OSTDEV $OST_MOUNT_OPTS; then
        error_noexit "mount $MMP_OSTDEV on $MMP_OSS should fail"
        stop $MMP_OSS || return ${PIPESTATUS[0]}
        return 2
    fi

    reset_mmp_block $MMP_OSS $MMP_OSTDEV || return ${PIPESTATUS[0]}
    return 0
}
run_test 9 "mount after aborted e2fsck"

# Test 10 - e2fsck with mounted filesystem.
test_10() {
    local rc=0

    log "Mounting $MMP_MDSDEV on $MMP_MDS..."
    start $MMP_MDS $MMP_MDSDEV $MDS_MOUNT_OPTS || return ${PIPESTATUS[0]}

    run_e2fsck $MMP_MDS_FAILOVER $MMP_MDSDEV "-fn"
    rc=${PIPESTATUS[0]}

    # e2fsck is called with -n option (Open the filesystem read-only), so
    # 0 (No errors) and 4 (File system errors left uncorrected) are the only
    # acceptable exit codes in this case
    if [ $rc -ne 0 ] && [ $rc -ne 4 ]; then
        error_noexit "e2fsck $MMP_MDSDEV on $MMP_MDS_FAILOVER returned $rc"
        stop $MMP_MDS || return ${PIPESTATUS[0]}
        return $rc
    fi

    log "Mounting $MMP_OSTDEV on $MMP_OSS..."
    start $MMP_OSS $MMP_OSTDEV $OST_MOUNT_OPTS
    rc=${PIPESTATUS[0]}
    if [ $rc -ne 0 ]; then
        stop $MMP_MDS || return ${PIPESTATUS[0]}
        return $rc
    fi

    run_e2fsck $MMP_OSS_FAILOVER $MMP_OSTDEV "-fn"
    rc=${PIPESTATUS[0]}
    if [ $rc -ne 0 ] && [ $rc -ne 4 ]; then
        error_noexit "e2fsck $MMP_OSTDEV on $MMP_OSS_FAILOVER returned $rc"
    fi

    CLEANUP_DM_DEV=true stop_services primary || return ${PIPESTATUS[0]}
    return 0
}
run_test 10 "e2fsck with mounted filesystem"

mmp_fini
FAIL_ON_ERROR=$SAVED_FAIL_ON_ERROR

complete $SECONDS
$MMP_RESTORE_MOUNT && setupall
check_and_cleanup_lustre
exit_status


#!/bin/bash
# this will cause debugfs to create the /tmp/debugfs.mark file once it has
# passed the MMP startup, then continue reading input until it is killed
MARKFILE=$(mktemp)
DEBUGFS=${DEBUGFS:-debugfs}
DEVICE=$1

rm -f $MARKFILE
echo "$DEBUGFS -w $DEVICE"
{ echo "dump_inode <2> $MARKFILE"; cat /dev/zero; } | $DEBUGFS -w $DEVICE &
debugfspid=$!
while [ ! -e $MARKFILE ]; do
        sleep 1
done
rm -f $MARKFILE
kill -9 $debugfspid


#!/bin/bash
#set -x
set -e

LUSTRE=${LUSTRE:-`dirname $0`/..}
. $LUSTRE/tests/test-framework.sh
init_test_env $@

nobjhi=${nobjhi:-1}
thrhi=${thrhi:-16}
size=${size:-1024}

# the summary file a bit smaller than OSTSIZE
. ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
init_logging

[ "$SLOW" = no ] && { nobjhi=1; thrhi=4; }
thrlo=${thrlo:-$(( thrhi / 2))}

# bug number for skipped test:
ALWAYS_EXCEPT="$OBDFILTER_SURVEY_EXCEPT"
# UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!

OBDSURVEY=${OBDSURVEY:-$(which obdfilter-survey)}

build_test_filter
check_and_setup_lustre

# FIXME: the summary file a bit smaller than OSTSIZE, add estimation
minsize=$(min_ost_size)
if [ $(( size * 1024 )) -ge $minsize  ]; then
    size=$((minsize * 10 / 1024 / 12 ))
    echo min kbytesavail: $minsize using size=${size} MBytes per obd instance
fi

get_targets () {
	local targets
	local target
	local dev
	local nid
	local osc

	for osc in $($LCTL get_param -N osc.${FSNAME}-*osc-*); do
		nid=$($LCTL get_param $osc.import |
			awk '/current_connection:/ {sub(/@.*/,""); print $2}')
		dev=$(echo $osc | sed -e 's/^osc\.//' -e 's/-osc.*//')
		target=$dev

		# For local disk obdfilter-survey requires target devs w/o nid.
		# obdfilter-survey :
		# case 1 (local disk):
		#    $ nobjhi=2 thrhi=2 size=1024
		#      targets="$nid:$FSNAME-OST0000 $nid:$FSNAME-OST0001 ..."
		#                 sh obdfilter-survey
		local_node && [ "$1" == "disk" ] || target=$nid:$target
		targets="$targets $target"
	done
	echo $targets
}

obdflter_survey_targets () {
	local case=$1
	local targets

	case $case in
		disk)    targets=$(get_targets $case);;
		netdisk) targets=$(get_targets $case);;
		network) targets=$(host_nids_address $(comma_list $(osts_nodes)) $NETTYPE);;
		*) error "unknown obdflter-survey case!" ;;
	esac
	echo $targets
}

obdflter_survey_run () {
	local case=$1

	rm -f ${TMP}/obdfilter_survey*

	local targets=$(obdflter_survey_targets $case)
	local cmd="NETTYPE=$NETTYPE thrlo=$thrlo nobjhi=$nobjhi thrhi=$thrhi size=$size case=$case rslt_loc=${TMP} targets=\"$targets\" $OBDSURVEY"
	echo + $cmd
	eval $cmd
	local rc=$?

	cat ${TMP}/obdfilter_survey*
	[ $rc = 0 ] || error "$OBDSURVEY failed: $rc"
}

test_1a () {
	obdflter_survey_run disk
}
run_test 1a "Object Storage Targets survey"

test_1b () {
	local param_file=$TMP/$tfile-params

	do_nodesv $(comma_list $(osts_nodes)) \
		$LCTL get_param obdfilter.${FSNAME}-*.sync_journal
	save_lustre_params $(get_facets OST) \
		"obdfilter.${FSNAME}-*.sync_journal" > $param_file
	do_nodesv $(comma_list $(osts_nodes)) \
		$LCTL set_param obdfilter.${FSNAME}-*.sync_journal=0

	local stime=$(date +%s)
	thrlo=4 nobjhi=1 thrhi=4 obdflter_survey_run disk
	local etime=$(date +%s)
	# run time of obd survey
	local rtime=$((etime - stime))

	echo "obd survey finished in $rtime seconds"
	restore_lustre_params < $param_file

	rm -f $param_file
}
run_test 1b "Object Storage Targets survey, async journal"

test_1c () {
	nobjlo=1 nobjhi=1 thrlo=32 thrhi=32 rszlo=1024 rszhi=1024 size=8192\
	obdflter_survey_run disk
}
run_test 1c "Object Storage Targets survey, big batch"

test_2a () {
	obdflter_survey_run netdisk
}
run_test 2a "Stripe F/S over the Network"

test_2b () {
	local param_file=$TMP/$tfile-params

	do_nodesv $(comma_list $(osts_nodes)) \
		$LCTL get_param obdfilter.${FSNAME}-*.sync_journal
	save_lustre_params $(get_facets OST) \
		"obdfilter.${FSNAME}-*.sync_journal" > $param_file
	do_nodesv $(comma_list $(osts_nodes)) \
		$LCTL set_param obdfilter.${FSNAME}-*.sync_journal=0

	local stime=$(date +%s)
	thrlo=4 nobjhi=1 thrhi=4 obdflter_survey_run netdisk
	local etime=$(date +%s)
	local rtime=$((etime - stime))

	echo "obd survey finished in $rtime seconds"
	restore_lustre_params < $param_file

	rm -f $param_file
}
run_test 2b "Stripe F/S over the Network, async journal"

# README.obdfilter-survey: In network test only automated run is supported.
test_3a () {
	# obdfilter-survey Prerequisite:
	#    For "network" case  you need to have all
	#    modules (those llmount.sh loads) loaded in kernel. And the
	#    'lctl dl' output must be blank.
	# Skipping test for CLIENTONLY mode because of
	# cleanupall()->stopall() does not cleanup the servers on this mode.
	[ "$CLIENTONLY" ] && skip "CLIENTONLY mode"

	remote_servers || skip "Local servers"

	cleanupall

	obdflter_survey_run network

	setupall
}
run_test 3a "Network survey"

complete $SECONDS
cleanup_echo_devs
check_and_cleanup_lustre
exit_status


#!/bin/bash

set -e
#set -vx

LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
. $LUSTRE/tests/test-framework.sh
. ${CONFIG:=$LUSTRE/tests/cfg/${NAME}.sh}

export PATH=`dirname $0`/../utils:$PATH
LFS=${LFS:-lfs}
LCTL=${LCTL:-lctl}
MOUNT=${MOUNT:-$1}
MOUNT=${MOUNT:-/mnt/lustre}
OOS=$MOUNT/oosfile
TMP=${TMP:-/tmp}
LOG=$TMP/$(basename $0 .sh).log

SUCCESS=1

rm -f $OOS $LOG

sync; sleep 1; sync	# to ensure we get up-to-date statfs info

#$LCTL set_param -n debug=-1
#$LCTL set_param -n subsystem_debug=0x40a8

#$LCTL clear
#$LCTL debug_daemon start /r/tmp/debug 1024

STRIPECOUNT=`$LCTL get_param -n lov.*.activeobd | head -n 1`
ORIGFREE=`$LCTL get_param -n llite.*.kbytesavail | head -n 1`
MAXFREE=${MAXFREE:-$((400000 * $STRIPECOUNT))}
echo STRIPECOUNT=$STRIPECOUNT ORIGFREE=$ORIGFREE MAXFREE=$MAXFREE
if [ $ORIGFREE -gt $MAXFREE ]; then
	skip "$0: ${ORIGFREE}kB free gt MAXFREE ${MAXFREE}kB, increase $MAXFREE (or reduce test fs size) to proceed"
	exit 0
fi

export LANG=C LC_LANG=C # for "No space left on device" message

[ -f $LOG ] && error "log file wasn't removed?"

echo BEFORE dd started
oos_full || true

# make sure we stripe over all OSTs to avoid OOS on only a subset of OSTs
$LFS setstripe $OOS -c $STRIPECOUNT
# add 20% of margin since the metadata overhead estimated in bavail might be
# too aggressive and we might be able to write more than reported initially
#Â by statfs.
echo dd size $((ORIGFREE * 120 / 100))kB
if dd if=/dev/zero of=$OOS count=$((ORIGFREE * 120 / 100)) bs=1k 2> $LOG; then
	echo "ERROR: dd did not fail"
	SUCCESS=0
fi

[ ! -s "$LOG" ] && error "LOG file is empty!"

if [ "`grep -c 'No space left on device' $LOG`" -ne 1 ]; then
	echo "ERROR: dd not return ENOSPC"
	sed "s/^/LOG: /" $LOG
	SUCCESS=0
fi

# flush cache to OST(s) so avail numbers are correct
sync; sleep 1 ; sync

echo AFTER dd
if ! oos_full; then
	echo "no OSTs are close to full"
	SUCCESS=0
fi

RECORDSOUT=`grep "records out" $LOG | cut -d + -f1`
FILESIZE=`ls -l $OOS | awk '{ print $5 }'`
if [ -z "$RECORDSOUT" ]; then
	echo "ERROR: no blocks written by dd?"
	sed "s/^/LOG: /" $LOG
	SUCCESS=0
elif [ "$RECORDSOUT" -ne $((FILESIZE / 1024)) ]; then
	echo "ERROR: blocks written by dd not equal to the size of file"
	SUCCESS=0
fi

#$LCTL debug_daemon stop

[ $SUCCESS != 0 ] && echo LOG file && sed "s/^/LOG: /" $LOG
rm -f $OOS

sync; sleep 3; sync

wait_delete_completed 300

if [ $SUCCESS -eq 1 ]; then
	echo "Success!"
	rm -f $LOG
else
	exit 1
fi


#!/bin/bash

set -e

LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
. $LUSTRE/tests/test-framework.sh
. ${CONFIG:=$LUSTRE/tests/cfg/${NAME}.sh}

export PATH=$LUSTRE/utils:$PATH
LFS=${LFS:-lfs}
LCTL=${LCTL:-lctl}
MOUNT=${MOUNT:-$1}
MOUNT=${MOUNT:-/mnt/lustre}
MOUNT2=${MOUNT2:-$2}
MOUNT2=${MOUNT2:-${MOUNT}2}
OOS=$MOUNT/oosfile
OOS2=$MOUNT2/oosfile2
TMP=${TMP:-/tmp}
LOG=$TMP/$(basename $0 .sh).log
LOG2=${LOG}2

SUCCESS=1

rm -f $OOS $OOS2 $LOG $LOG2

sync; sleep 1; sync	# to ensure we get up-to-date statfs info

STRIPECOUNT=`$LCTL get_param -n lov.*.activeobd | head -n 1`
ORIGFREE=`$LCTL get_param -n llite.*.kbytesavail | head -n 1`
MAXFREE=${MAXFREE:-$((400000 * $STRIPECOUNT))}
echo STRIPECOUNT=$STRIPECOUNT ORIGFREE=$ORIGFREE MAXFREE=$MAXFREE
if [ $ORIGFREE -gt $MAXFREE ]; then
	skip "$0: ${ORIGFREE}kB free gt MAXFREE ${MAXFREE}kB, increase $MAXFREE (or reduce test fs size) to proceed"
	exit 0
fi

export LANG=C LC_LANG=C # for "No space left on device" message

# make sure we stripe over all OSTs to avoid OOS on only a subset of OSTs
$LFS setstripe $OOS -c $STRIPECOUNT
$LFS setstripe $OOS2 -c $STRIPECOUNT
dd if=/dev/zero of=$OOS count=$((3 * $ORIGFREE / 4 + 100)) bs=1k 2>> $LOG &
DDPID=$!
if dd if=/dev/zero of=$OOS2 count=$((3*$ORIGFREE/4 + 100)) bs=1k 2>> $LOG2; then
	echo "ERROR: dd2 did not fail"
	SUCCESS=0
fi
if wait $DDPID; then
	echo "ERROR: dd did not fail"
	SUCCESS=0
fi

[ ! -s "$LOG" ] && error "LOG file is empty!"
[ ! -s "$LOG2" ] && error "LOG2 file is empty!"

if [ "`cat $LOG $LOG2 | grep -c 'No space left on device'`" -ne 2 ]; then
	echo "ERROR: dd not return ENOSPC"
	SUCCESS=0
fi

# flush cache to OST(s) so avail numbers are correct
sync; sleep 1 ; sync

if ! oos_full; then
	echo "no OSTs are close to full"
	SUCCESS=0
fi

RECORDSOUT=$((`grep "records out" $LOG | cut -d+ -f 1` + \
              `grep "records out" $LOG2 | cut -d+ -f 1`))

FILESIZE=$((`ls -l $OOS | awk '{print $5}'` + `ls -l $OOS2 | awk '{print $5}'`))
if [ "$RECORDSOUT" -ne $(($FILESIZE / 1024)) ]; then
        echo "ERROR: blocks written by dd not equal to the size of file"
        SUCCESS=0
fi

echo LOG LOG2 file
cat $LOG $LOG2

rm -f $OOS $OOS2
sync; sleep 1; sync

if [ $SUCCESS -eq 1 ]; then
	echo "Success!"
	rm -f $LOG $LOG2
else
	exit 1
fi