#!/bin/bash
# -*- mode: Bash; tab-width: 4; indent-tabs-mode: t; -*-
# vim:shiftwidth=4:softtabstop=4:tabstop=4:
#
# NAME
#
#   ha.sh - test Lustre HA (aka failover) configurations
#
# SYNOPSIS
#
#   ha.sh [OPTIONS]
#
# DESCRIPTION
#
#   ha.sh tests Lustre HA (aka failover) configurations with a CRM.
#
# OPTIONS
#
#   -h
#       Help.
#
#   -c HOST[,...]
#       Specify client nodes.
#
#   -s HOST[,...]
#       Specify server nodes.
#
#   -v HOST[,...]
#       Specify victim nodes to be rebooted.
#
#   -d DIRECTORY
#       Choose a parent of the test directory.  "/mnt/lustre" if not specified.
#
#   -u SECONDS
#       Define a duration for the test. 86400 seconds if not specified.
#
#   -p SECONDS
#       Define a max failover period. 10 minutes if not set.
#
#   -w
#       Only run the workloads; no failure will be introduced.
#       -v, -s are ignored in this case.
#   -r
#       Workloads dry run for several seconds; no failures will be introduced.
#       This option is useful to verify the loads.
#       -u is ignored in this case
#   -m
#       Reboot victim nodes simultaneously.
#
#
# ASSUMPTIONS
#
#   A Lustre file system is up and mounted on all client nodes.  This script
#   does not mount or unmount any Lustre targets or clients, let alone format
#   anything.
#
#   Each target has a failnode, so that workloads can continue after a power
#   failure.
#
#   CRM could be configured by 2 ways:
#   1.
#   Targets are automatically failed back when their primary node is back.  This
#   assumption avoids calling CRM-specific commands to trigger failbacks, making
#   this script more CRM-neural.
#   2.
#   Targets are not automatically failed back when their primary node is back.
#   CRM-specific command is executed to trigger failbacks.
#
#   A crash dump mechanism is configured to catch LBUGs, panics, etc.
#
# WORKLOADS
#
#   Each client runs set of MPI and non-MPI workloads. These
#   applications are run in short loops so that their exit status can be waited
#   for and checked within reasonable time by ha_wait_loads.
#   The set of MPI and non-MPI workloads are configurable by parameters:
#	ha_nonmpi_loads
#		default set: dd, tar, iozone
#	ha_mpi_loads
#		default set: ior, simul, mdtest
#
#   The number of clients run MPI loads is configured by parameter
#   ha_mpi_instances. Only one client runs MPI workloads by default.
#
#   MPI workloads can be run from several users. The list of users to use is
#   configured by parameter ha_mpi_users, default is "mpiuser".
#
# PROCESS STRUCTURE AND IPC
#
#   On the node where this script is run, the processes look like this:
#
#       ~ ha.sh (ha_killer)
#
#           ~ ha.sh (ha_repeat_mpi_load ior)
#               ~ mpirun IOR
#           ~ ha.sh (ha_repeat_mpi_load simul)
#               ~ mpirun simul
#           ~ ha.sh (ha_repeat_mpi_load mdtest)
#               ~ mpirun mdtest
#           ~ ... (one for each MPI load)
#
#           ~ ha.sh (ha_repeat_nonmpi_load client2 dbench)
#               ~ pdsh client2 dbench
#           ~ ha.sh (ha_repeat_nonmpi_load client2 iozone)
#               ~ pdsh client2 iozone
#           ~ ha.sh (ha_repeat_nonmpi_load client5 iozone)
#               ~ pdsh client5 iozone
#           ~ ... (one for each non-MPI load on each client)
#
#   Each tilde represents a process.  Indentations imply parent-children
#   relation.
#
#   IPC is done by files in the temporary directory.
#

#set -x

SIMUL=${SIMUL:-$(which simul 2> /dev/null || true)}
IOR=${IOR:-$(which IOR 2> /dev/null || true)}
MDTEST=${MDTEST:-$(which mdtest 2> /dev/null || true)}

ior_blockSize=${ior_blockSize:-6g}
mpi_threads_per_client=${mpi_threads_per_client:-2}

iozone_SIZE=${iozone_SIZE:-262144} # 256m

mpirun=${MPIRUN:-$(which mpirun)}
LFS=${LFS:-$(which lfs)}

ha_check_env()
{
	for ((load = 0; load < ${#ha_mpi_load_tags[@]}; load++)); do
		local tag=${ha_mpi_load_tags[$load]}
		local bin=$(echo $tag | tr '[:lower:]' '[:upper:]')
		if [ x${!bin} = x ]; then
			ha_error ha_mpi_loads: ${ha_mpi_loads}, $bin is not set
			exit 1
		fi
	done
}

ha_info()
{
	echo "$0: $(date +%H:%M:%S' '%s):" "$@"
}

ha_log()
{
	local nodes=${1// /,}
	shift
	ha_on $nodes "lctl mark $*"
}

ha_error()
{
    ha_info "$@" >&2
}

ha_trap_err()
{
    local i

    ha_error "Trap ERR triggered by:"
    ha_error "    $BASH_COMMAND"
    ha_error "Call trace:"
    for ((i = 0; i < ${#FUNCNAME[@]}; i++)); do
        ha_error "    ${FUNCNAME[$i]} [${BASH_SOURCE[$i]}:${BASH_LINENO[$i]}]"
    done
}

trap ha_trap_err ERR
set -eE

declare     ha_tmp_dir=/tmp/$(basename $0)-$$
declare     ha_stop_file=$ha_tmp_dir/stop
declare     ha_fail_file=$ha_tmp_dir/fail
declare     ha_status_file_prefix=$ha_tmp_dir/status
declare -a  ha_status_files
declare     ha_machine_file=$ha_tmp_dir/machine_file
declare     ha_lfsck_log=$ha_tmp_dir/lfsck.log
declare     ha_lfsck_lock=$ha_tmp_dir/lfsck.lock
declare     ha_lfsck_stop=$ha_tmp_dir/lfsck.stop
declare     ha_lfsck_bg=${LFSCK_BG:-false}
declare     ha_lfsck_after=${LFSCK_AFTER:-false}
declare     ha_lfsck_node=${LFSCK_NODE:-""}
declare     ha_lfsck_device=${LFSCK_DEV:-""}
declare     ha_lfsck_types=${LFSCK_TYPES:-"namespace layout"}
declare     ha_lfsck_custom_params=${LFSCK_CUSTOM_PARAMS:-""}
declare     ha_lfsck_wait=${LFSCK_WAIT:-1200}
declare     ha_lfsck_fail_on_repaired=${LFSCK_FAIL_ON_REPAIRED:-false}
declare     ha_power_down_cmd=${POWER_DOWN:-"pm -0"}
declare     ha_power_up_cmd=${POWER_UP:-"pm -1"}
declare     ha_power_delay=${POWER_DELAY:-60}
declare     ha_failback_delay=${DELAY:-5}
declare     ha_failback_cmd=${FAILBACK:-""}
declare     ha_stripe_params=${STRIPEPARAMS:-"-c 0"}
declare     ha_dir_stripe_count=${DSTRIPECOUNT:-"1"}
declare     ha_mdt_index=${MDTINDEX:-"0"}
declare     ha_mdt_index_random=${MDTINDEXRAND:-false}
declare -a  ha_clients
declare -a  ha_servers
declare -a  ha_victims
declare     ha_test_dir=/mnt/lustre/$(basename $0)-$$
declare     ha_start_time=$(date +%s)
declare     ha_expected_duration=$((60 * 60 * 24))
declare     ha_max_failover_period=10
declare     ha_nr_loops=0
declare     ha_stop_signals="SIGINT SIGTERM SIGHUP"
declare     ha_load_timeout=$((60 * 10))
declare     ha_workloads_only=false
declare     ha_workloads_dry_run=false
declare     ha_simultaneous=false

declare     ha_mpi_instances=${ha_mpi_instances:-1}

declare     ha_mpi_loads=${ha_mpi_loads="ior simul mdtest"}
declare -a  ha_mpi_load_tags=($ha_mpi_loads)
declare -a  ha_mpiusers=(${ha_mpi_users="mpiuser"})

declare     ha_ior_params=${IORP:-'" -b $ior_blockSize -t 2m -w -W -T 1"'}
declare     ha_simul_params=${SIMULP:-'" -n 10"'}
declare     ha_mdtest_params=${MDTESTP:-'" -i 1 -n 1000"'}
declare     ha_mpirun_options=${MPIRUN_OPTIONS:-""}
declare     ha_clients_stripe=${CLIENTSSTRIPE:-'"$STRIPEPARAMS"'}
declare     ha_nclientsset=${NCLIENTSSET:-1}
declare     ha_ninstmustfail=${NINSTMUSTFAIL:-0}

declare     ha_racer_params=${RACERP:-"MDSCOUNT=1"}

eval ha_params_ior=($ha_ior_params)
eval ha_params_simul=($ha_simul_params)
eval ha_params_mdtest=($ha_mdtest_params)
eval ha_stripe_clients=($ha_clients_stripe)

declare ha_nparams_ior=${#ha_params_ior[@]}
declare ha_nparams_simul=${#ha_params_simul[@]}
declare ha_nparams_mdtest=${#ha_params_mdtest[@]}
declare ha_nstripe_clients=${#ha_stripe_clients[@]}

declare -A  ha_mpi_load_cmds=(
	[ior]="$IOR -o {}/f.ior {params}"
	[simul]="$SIMUL {params} -d {}"
	[mdtest]="$MDTEST {params} -d {}"
)

declare racer=${RACER:-"$(dirname $0)/racer/racer.sh"}

declare     ha_nonmpi_loads=${ha_nonmpi_loads="dd tar iozone"}
declare -a  ha_nonmpi_load_tags=($ha_nonmpi_loads)
declare -A  ha_nonmpi_load_cmds=(
	[dd]="dd if=/dev/zero of={}/f.dd bs=1M count=256"
	[tar]="tar cf - /etc | tar xf - -C {}"
	[iozone]="iozone -a -e -+d -s $iozone_SIZE {}/f.iozone"
	[racer]="$ha_racer_params $racer {}"
)

ha_usage()
{
    ha_info "Usage: $0 -c HOST[,...] -s HOST[,...]"                         \
            "-v HOST[,...] [-d DIRECTORY] [-u SECONDS]"
}

ha_process_arguments()
{
    local opt

	while getopts hc:s:v:d:p:u:wrm opt; do
        case $opt in
        h)
            ha_usage
            exit 0
            ;;
        c)
            ha_clients=(${OPTARG//,/ })
            ;;
        s)
            ha_servers=(${OPTARG//,/ })
            ;;
        v)
            ha_victims=(${OPTARG//,/ })
            ;;
        d)
            ha_test_dir=$OPTARG/$(basename $0)-$$
            ;;
        u)
            ha_expected_duration=$OPTARG
            ;;
	p)
		ha_max_failover_period=$OPTARG
		;;
        w)
		ha_workloads_only=true
		;;
	r)
		ha_workloads_dry_run=true
		;;
	m)
		ha_simultaneous=true
		;;
        \?)
            ha_usage
            exit 1
            ;;
        esac
    done

	if [ -z "${ha_clients[*]}" ]; then
		ha_error "-c is mandatory"
		ha_usage
		exit 1
	fi
	if ! ($ha_workloads_dry_run ||
			$ha_workloads_only) &&
			([ -z "${ha_servers[*]}" ] ||
			[ -z "${ha_victims[*]}" ]); then
		ha_error "-s, and -v are all mandatory"
		ha_usage
		exit 1
	fi
}

ha_on()
{
	local nodes=$1
	local rc=0

	shift

	#
	# -S is to be used here to track the
	# remote command return values
	#
	pdsh -S -w $nodes PATH=/usr/local/sbin:/usr/local/bin:/sbin:\
/bin:/usr/sbin:/usr/bin "$@" ||
		rc=$?
	return $rc
}

ha_trap_exit()
{
	touch "$ha_stop_file"
	trap 0
	if [ -e "$ha_fail_file" ]; then
		ha_info "Test directory $ha_test_dir not removed"
		ha_info "Temporary directory $ha_tmp_dir not removed"
	else
		ha_on ${ha_clients[0]} rm -rf "$ha_test_dir"
		ha_info "Please find the results in the directory $ha_tmp_dir"
	fi
}

ha_trap_stop_signals()
{
    ha_info "${ha_stop_signals// /,} received"
    touch "$ha_stop_file"
}

ha_sleep()
{
    local n=$1

    ha_info "Sleeping for ${n}s"
    #
    # sleep(1) could interrupted.
    #
    sleep $n || true
}

ha_wait_unlock()
{
	local lock=$1

	while [ -e $lock ]; do
		sleep 1
	done
}

ha_lock()
{
    local lock=$1

    until mkdir "$lock" >/dev/null 2>&1; do
        ha_sleep 1 >/dev/null
    done
}

ha_unlock()
{
    local lock=$1

    rm -r "$lock"
}

ha_dump_logs()
{
	local nodes=${1// /,}
	local file=/tmp/$(basename $0)-$$-$(date +%s).dk
	local lock=$ha_tmp_dir/lock-dump-logs
	local rc=0

	ha_lock "$lock"
	ha_info "Dumping lctl log to $file"

	#
	# some nodes could crash, so
	# do not exit with error if not all logs are dumped
	#
	ha_on $nodes "lctl dk >>$file" || rc=$?

	[ $rc -eq 0 ] ||
		ha_error "not all logs are dumped! Some nodes are unreachable."
	ha_unlock "$lock"
}

ha_repeat_mpi_load()
{
	local client=$1
	local load=$2
	local status=$3
	local parameter=$4
	local machines=$5
	local stripeparams=$6
	local mpiuser=$7
	local mustpass=$8
	local tag=${ha_mpi_load_tags[$load]}
	local cmd=${ha_mpi_load_cmds[$tag]}
	local dir=$ha_test_dir/$client-$tag
	local log=$ha_tmp_dir/$client-$tag
	local rc=0
	local nr_loops=0
	local avg_loop_time=0
	local start_time=$(date +%s)

	cmd=${cmd//"{}"/$dir}
	cmd=${cmd//"{params}"/$parameter}

	[[ -n "$ha_postcmd" ]] && ha_postcmd=${ha_postcmd//"{}"/$dir}
	ha_info "Starting $tag"

	machines="-machinefile $machines"
	while [ ! -e "$ha_stop_file" ] && ((rc == 0)); do
		ha_info "$client Starts: $cmd" 2>&1 |  tee -a $log
		{
		local mdt_index
		if $ha_mdt_index_random && [ $ha_mdt_index -ne 0 ]; then
			mdt_index=$(ha_rand $ha_mdt_index)
		else
			mdt_index=$ha_mdt_index
		fi
		ha_on $client $LFS mkdir -i$mdt_index -c$ha_dir_stripe_count "$dir" &&
		ha_on $client $LFS getdirstripe "$dir" &&
		ha_on $client $LFS setstripe $stripeparams $dir &&
		ha_on $client $LFS getstripe $dir &&
		ha_on $client chmod a+xwr $dir &&
		ha_on $client "su $mpiuser sh -c \" $mpirun $ha_mpirun_options \
			-np $((${#ha_clients[@]} * mpi_threads_per_client )) \
			$machines $cmd \" " || rc=$?
		[[ -n "$ha_postcmd" ]] && ha_info "$ha_postcmd" &&
			ha_on $client $ha_postcmd >>"$log" 2>&1
		(( ((rc == 0)) && (( mustpass != 0 )) )) ||
		(( ((rc != 0)) && (( mustpass == 0 )) )) &&
			ha_on $client rm -rf "$dir";
		} >>"$log" 2>&1 || rc=$?

		ha_info rc=$rc mustpass=$mustpass

		# mustpass=0 means that failure is expected
		if (( rc !=0 )); then
			if (( mustpass != 0 )); then
				touch "$ha_fail_file"
				touch "$ha_stop_file"
				ha_dump_logs "${ha_clients[*]} ${ha_servers[*]}"
			else
				# Ok to fail
				rc=0
			fi
		elif (( mustpass == 0 )); then
			touch "$ha_fail_file"
			touch "$ha_stop_file"
			ha_dump_logs "${ha_clients[*]} ${ha_servers[*]}"
		fi
		echo rc=$rc mustpass=$mustpass >"$status"

		nr_loops=$((nr_loops + 1))
	done

	[ $nr_loops -ne 0 ] &&
		avg_loop_time=$((($(date +%s) - start_time) / nr_loops))

	ha_info "$tag stopped: rc=$rc mustpass=$mustpass \
		avg loop time $avg_loop_time"
}

ha_start_mpi_loads()
{
	local client
	local load
	local tag
	local status
	local n
	local nparam
	local machines
	local m
	local -a mach
	local mpiuser

	# ha_mpi_instances defines the number of
	# clients start mpi loads; should be <= ${#ha_clients[@]}
	# do nothing if
	#    ha_mpi_instances = 0
	# or
	#    ${#ha_mpi_load_tags[@]} =0
	local inst=$ha_mpi_instances
	(( inst == 0 )) || (( ${#ha_mpi_load_tags[@]} == 0 )) &&
		ha_info "no mpi load to start" &&
		return 0

	(( inst <= ${#ha_clients[@]} )) || inst=${#ha_clients[@]}

	# Define names for machinefiles for each client set
	for (( n=0; n < $ha_nclientsset; n++ )); do
		mach[$n]=$ha_machine_file$n
	done

	for ((n = 0; n < ${#ha_clients[@]}; n++)); do
		m=$(( n % ha_nclientsset))
		machines=${mach[m]}
		ha_info machine_file=$machines
		echo ${ha_clients[n]} >> $machines
	done
	local dirname=$(dirname $ha_machine_file)
	for client in ${ha_clients[@]}; do
		ha_on $client mkdir -p $dirname
		scp $ha_machine_file* $client:$dirname
	done

	for ((n = 0; n < $inst; n++)); do
		client=${ha_clients[n]}
		mpiuser=${ha_mpiusers[$((n % ${#ha_mpiusers[@]}))]}
		for ((load = 0; load < ${#ha_mpi_load_tags[@]}; load++)); do
			tag=${ha_mpi_load_tags[$load]}
			status=$ha_status_file_prefix-$tag-$client
			# ha_nparams_ior
			# ha_nparams_simul
			local num=ha_nparams_$tag
			nparam=$((n % num))
			local aref=ha_params_$tag[nparam]
			local parameter=${!aref}
			local nstripe=$((n % ha_nstripe_clients))
			aref=ha_stripe_clients[nstripe]
			local stripe=${!aref}
			local m=$(( n % ha_nclientsset))
			machines=${mach[m]}
			local mustpass=1
			[[ $ha_ninstmustfail == 0 ]] ||
				mustpass=$(( n % ha_ninstmustfail ))
			ha_repeat_mpi_load $client $load $status "$parameter" \
				$machines "$stripe" "$mpiuser" "$mustpass" &
				ha_status_files+=("$status")
		done
	done
}

ha_repeat_nonmpi_load()
{
	local client=$1
	local load=$2
	local status=$3
	local tag=${ha_nonmpi_load_tags[$load]}
	local cmd=${ha_nonmpi_load_cmds[$tag]}
	local dir=$ha_test_dir/$client-$tag
	local log=$ha_tmp_dir/$client-$tag
	local rc=0
	local nr_loops=0
	local avg_loop_time=0
	local start_time=$(date +%s)

	cmd=${cmd//"{}"/$dir}

	ha_info "Starting $tag on $client"

	while [ ! -e "$ha_stop_file" ] && ((rc == 0)); do
		ha_info "$client Starts: $cmd" 2>&1 |  tee -a $log
		ha_on $client "mkdir -p $dir &&                              \
			$cmd &&                                              \
			rm -rf $dir" >>"$log" 2>&1 || rc=$?

		if ((rc != 0)); then
			ha_dump_logs "${ha_clients[*]} ${ha_servers[*]}"
			touch "$ha_fail_file"
			touch "$ha_stop_file"
		fi
		echo $rc >"$status"

		nr_loops=$((nr_loops + 1))
	done

	[ $nr_loops -ne 0 ] &&
		avg_loop_time=$((($(date +%s) - start_time) / nr_loops))

	ha_info "$tag on $client stopped: rc $rc avg loop time ${avg_loop_time}s"
}

ha_start_nonmpi_loads()
{
    local client
    local load
    local tag
    local status

    for client in ${ha_clients[@]}; do
        for ((load = 0; load < ${#ha_nonmpi_load_tags[@]}; load++)); do
            tag=${ha_nonmpi_load_tags[$load]}
            status=$ha_status_file_prefix-$tag-$client
            ha_repeat_nonmpi_load $client $load $status &
            ha_status_files+=("$status")
        done
    done
}

ha_lfsck_bg () {
	rm -f $ha_lfsck_log
	rm -f $ha_lfsck_stop

	ha_info "LFSCK BG"
	while [ true ]; do
		[ -f $ha_lfsck_stop ] && ha_info "LFSCK stopped" && break
		[ -f $ha_stop_file ] &&
			ha_info "$ha_stop_file found! LFSCK not started" &&
			break
		ha_start_lfsck 2>&1 | tee -a $ha_lfsck_log
		sleep 1
	done &
	LFSCK_BG_PID=$!
	ha_info LFSCK BG PID: $LFSCK_BG_PID
}

ha_wait_lfsck_completed () {
	local -a status
	local -a types=($ha_lfsck_types)
	local type
	local s

	local nodes="${ha_servers[@]}"
	nodes=${nodes// /,}

	# -A start LFSCK on all nodes
	# -t default all
	[ ${#types[@]} -eq 0 ] && types=(namespace layout)
	ha_info "Waiting LFSCK completed in $ha_lfsck_wait sec: types ${types[@]}"
	for type in ${types[@]}; do
		eval var_$type=0
		for (( i=0; i<=ha_lfsck_wait; i++)); do
			status=($(ha_on $nodes lctl get_param -n *.*.lfsck_$type 2>/dev/null | \
				awk '/status/ { print $3 }'))
			for (( s=0; s<${#status[@]}; s++ )); do
				# "partial" is expected after HARD failover
				[[ "${status[s]}" = "completed" ]] ||
				[[ "${status[s]}" = "partial" ]] ||  break
			done
			[[ $s -eq ${#status[@]} ]] && eval var_$type=1 && break
			sleep 1
		done
		ha_info "LFSCK $type status in $i sec:"
		ha_on $nodes lctl get_param -n *.*.lfsck_$type 2>/dev/null | grep status

	done

	for type in ${types[@]}; do
		local var=var_$type
		ha_on $nodes lctl get_param -n *.*.lfsck_$type 2>/dev/null
		[[ ${!var} -eq 1 ]] ||
			{ ha_info "lfsck not completed in $ha_lfsck_wait sec";
			return 1; }
	done
	return 0
}

ha_start_lfsck()
{
	local -a types=($ha_lfsck_types)
	local rc=0

	# -A: start LFSCK on all nodes via the specified MDT device
	# (see "-M" option) by single LFSCK command
	local params=" -A -r $ha_lfsck_custom_params"

	# use specified device if set
	[ -n "$ha_lfsck_device" ] && params="-M $ha_lfsck_device $params"

	# -t: check type(s) to be performed (default all)
	# check only specified types if set
	if [ ${#types[@]} -ne 0 ]; then
		local type="${types[@]}"
		params="$params -t ${type// /,}"
	fi

	ha_info "LFSCK start $params"
	ha_on $ha_lfsck_node "lctl lfsck_start $params" || rc=1
	if [ $rc -ne 0 ]; then
		if [ -e $ha_lfsck_lock ]; then
			rc=0
			ha_wait_unlock $ha_lfsck_lock
			ha_sleep 120
			ha_on $ha_lfsck_node "lctl lfsck_start $params" || rc=1
		fi
	fi

	[ $rc -eq 0 ] ||
		{ touch "$ha_fail_file"; touch "$ha_stop_file";
		touch $ha_lfsck_stop; return 1; }

	ha_wait_lfsck_completed ||
		{ touch "$ha_fail_file"; touch "$ha_stop_file";
		touch $ha_lfsck_stop; return 1; }

	return 0
}

ha_lfsck_repaired()
{
	local n=0

	n=$(cat $ha_lfsck_log | awk '/repaired/ {print $3}' |\
		awk '{sum += $1} END { print sum }')
	[ $n -eq 0] ||
		{ ha_info "Total repaired: $n";
		touch "$ha_fail_file"; return 1; }
	return 0
}

ha_start_loads()
{
	$ha_lfsck_bg && ha_lfsck_bg
	trap ha_trap_stop_signals $ha_stop_signals
	ha_start_nonmpi_loads
	ha_start_mpi_loads
}

ha_stop_loads()
{
	touch $ha_stop_file
	# true because of lfsck_bg could be stopped already
	$ha_lfsck_bg && wait $LFSCK_BG_PID || true
	trap - $ha_stop_signals
	ha_info "Waiting for workloads to stop"
	wait
}

ha_wait_loads()
{
    local file
    local end=$(($(date +%s) + ha_load_timeout))

    ha_info "Waiting for workload status"
    rm -f "${ha_status_files[@]}"

	#
	# return immediately if ha_stop_file exists,
	# all status_files not needed to be checked
	#
	for file in "${ha_status_files[@]}"; do
		if [ -e "$ha_stop_file" ]; then
			ha_info "$ha_stop_file found! Stop."
			break
		fi
		#
		# Wait status file created during ha_load_timeout.
		# Existing file guarantees that some application
		# is completed. If no status file was created
		# this function guarantees that we allow
		# applications to continue after/before
		# failover/failback during ha_load_timeout time.
		#
		until [ -e "$file" ] || (($(date +%s) >= end)); do
			#
			# check ha_stop_file again, it could appear
			# during ha_load_timeout
			#
			if [ -e "$ha_stop_file" ]; then
				ha_info "$ha_stop_file found! Stop."
				break
			fi
			ha_sleep 1 >/dev/null
		done
	done
}

ha_power_down()
{
	local nodes=$1
	local rc=1
	local i

	if $ha_lfsck_bg && [[ ${nodes//,/ /} =~ $ha_lfsck_node ]]; then
		ha_info "$ha_lfsck_node down, delay start LFSCK"
		ha_lock $ha_lfsck_lock
	fi

	ha_info "Powering down $nodes"
	for i in $(seq 1 5); do
		$ha_power_down_cmd $nodes && rc=0 && break
		sleep $ha_power_delay
	done

	[ $rc -eq 0 ] || ha_info "Failed Powering down in $i attempts"
}

ha_power_up()
{
	local nodes=$1
	local rc=1
	local i

	ha_info "Powering up $nodes"
	for i in $(seq 1 5); do
		$ha_power_up_cmd $nodes && rc=0 && break
		sleep $ha_power_delay
	done

	[ $rc -eq 0 ] || ha_info "Failed Powering up in $i attempts"
}

#
# rand MAX
#
# Print a random integer within [0, MAX).
#
ha_rand()
{
    local max=$1

    #
    # See "5.2 Bash Variables" from "info bash".
    #
    echo -n $((RANDOM * max / 32768))
}

ha_aim()
{
	local i
	local nodes

	if $ha_simultaneous ; then
		nodes=$(echo ${ha_victims[@]})
		nodes=${nodes// /,}
	else
		i=$(ha_rand ${#ha_victims[@]})
		nodes=${ha_victims[$i]}
	fi

	echo -n $nodes
}

ha_wait_nodes()
{
	local nodes=$1
	local end=$(($(date +%s) + 10 * 60))

	ha_info "Waiting for $nodes to boot up"
	until ha_on $nodes hostname >/dev/null 2>&1 ||
		[ -e "$ha_stop_file" ] ||
			(($(date +%s) >= end)); do
		ha_sleep 1 >/dev/null
	done
}

ha_failback()
{
	local nodes=$1
	ha_info "Failback resources on $nodes in $ha_failback_delay sec"

	ha_sleep $ha_failback_delay
	[ "$ha_failback_cmd" ] ||
	{
		ha_info "No failback command set, skiping"
		return 0
	}

	$ha_failback_cmd $nodes
	[ -e $ha_lfsck_lock ] && ha_unlock $ha_lfsck_lock || true
}

ha_summarize()
{
    ha_info "---------------8<---------------"
    ha_info "Summary:"
    ha_info "    Duration: $(($(date +%s) - $ha_start_time))s"
    ha_info "    Loops: $ha_nr_loops"
}

ha_killer()
{
	local nodes

	while (($(date +%s) < ha_start_time + ha_expected_duration)) &&
			[ ! -e "$ha_stop_file" ]; do
		ha_info "---------------8<---------------"

		$ha_workloads_only || nodes=$(ha_aim)

		ha_info "Failing $nodes"
		$ha_workloads_only && ha_info "    is skipped: workload only..."

		ha_sleep $(ha_rand $ha_max_failover_period)
		$ha_workloads_only || ha_power_down $nodes
		ha_sleep 10
		ha_wait_loads || return

		if [ -e $ha_stop_file ]; then
			$ha_workloads_only || ha_power_up $nodes
			break
		fi

		ha_info "Bringing $nodes back"
		ha_sleep $(ha_rand 10)
		$ha_workloads_only ||
		{
			ha_power_up $nodes
			ha_wait_nodes $nodes
			ha_failback $nodes
		}

		#
		# Wait for the failback to start.
		#
		ha_sleep 60
		ha_wait_loads || return

		ha_sleep $(ha_rand 20)

		ha_nr_loops=$((ha_nr_loops + 1))
		ha_info "Loop $ha_nr_loops done"
	done
	ha_summarize
}

ha_main()
{
	ha_process_arguments "$@"
	ha_check_env

	ha_log "${ha_clients[*]} ${ha_servers[*]}" \
		"START: $0: $(date +%H:%M:%S' '%s)"
	trap ha_trap_exit EXIT
	mkdir "$ha_tmp_dir"
	ha_on ${ha_clients[0]} mkdir "$ha_test_dir"
	ha_on ${ha_clients[0]} " \
		$LFS setstripe $ha_stripe_params $ha_test_dir"

	ha_start_loads
	ha_wait_loads

	if $ha_workloads_dry_run; then
		ha_sleep 5
	else
		ha_killer
		ha_dump_logs "${ha_clients[*]} ${ha_servers[*]}"
	fi

	ha_stop_loads

	$ha_lfsck_after && ha_start_lfsck | tee -a $ha_lfsck_log

	$ha_lfsck_fail_on_repaired && ha_lfsck_repaired

	if [ -e "$ha_fail_file" ]; then
		exit 1
	else
		ha_log "${ha_clients[*]} ${ha_servers[*]}" \
			"END: $0: $(date +%H:%M:%S' '%s)"
		exit 0
	fi
}

ha_main "$@"


#!/bin/bash
#
# Test different failure modes combinations

set -e

LUSTRE=${LUSTRE:-$(dirname $0)/..}
. $LUSTRE/tests/test-framework.sh
init_test_env $@
init_logging

# bug number for skipped test:
ALWAYS_EXCEPT="$INSANITY_EXCEPT"
# UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!

build_test_filter

if [ "$FAILURE_MODE" = "HARD" ]; then
	skip_env "$TESTSUITE: is not functional with FAILURE_MODE = HARD, " \
		"please use recovery-double-scale, bz20407"
fi

SINGLECLIENT=${SINGLECLIENT:-$HOSTNAME}
LIVE_CLIENT=${LIVE_CLIENT:-$SINGLECLIENT}
FAIL_CLIENTS=${FAIL_CLIENTS:-$RCLIENTS}

assert_env mds_HOST MDSCOUNT
assert_env ost_HOST OSTCOUNT
assert_env LIVE_CLIENT FSNAME

require_dsh_mds || exit 0
require_dsh_ost || exit 0

# FAIL_CLIENTS list should not contain the LIVE_CLIENT
FAIL_CLIENTS=$(echo " $FAIL_CLIENTS " | sed -re "s/\s+$LIVE_CLIENT\s+/ /g")

DIR=${DIR:-$MOUNT}
TESTDIR=$DIR/d0.$TESTSUITE

#####
# fail clients round robin

# list of failable clients
FAIL_LIST=($FAIL_CLIENTS)
FAIL_NUM=${#FAIL_LIST[*]}
FAIL_NEXT=0
typeset -i  FAIL_NEXT
DOWN_NUM=0   # number of nodes currently down

# set next client to fail
set_fail_client() {
	FAIL_CLIENT=${FAIL_LIST[$FAIL_NEXT]}
	FAIL_NEXT=$(( (FAIL_NEXT+1) % FAIL_NUM ))
	echo "fail $FAIL_CLIENT, next is $FAIL_NEXT"
}

fail_clients() {
	num=$1

	log "Request fail clients: $num, to fail: $FAIL_NUM, failed: $DOWN_NUM"
	if [ -z "$num"  ] || [ "$num" -gt $((FAIL_NUM - DOWN_NUM)) ]; then
		num=$((FAIL_NUM - DOWN_NUM))
	fi

	if [ -z "$num" ] || [ "$num" -le 0 ]; then
		log "No clients failed!"
		return
	fi

	client_mkdirs

	for i in `seq $num`; do
		set_fail_client
		client=$FAIL_CLIENT
		DOWN_CLIENTS="$DOWN_CLIENTS $client"
		shutdown_client $client
	done

	echo "down clients: $DOWN_CLIENTS"

	for client in $DOWN_CLIENTS; do
		boot_node $client
	done
	DOWN_NUM=`echo $DOWN_CLIENTS | wc -w`
	client_rmdirs
}

reintegrate_clients() {
	for client in $DOWN_CLIENTS; do
		wait_for_host $client
		echo "Restarting $client"
		zconf_mount $client $MOUNT || return 1
	done

	DOWN_CLIENTS=""
	DOWN_NUM=0
}

start_ost() {
	start ost$1 `ostdevname $1` $OST_MOUNT_OPTS
}

start_mdt() {
	start mds$1 $(mdsdevname $1) $MDS_MOUNT_OPTS
}

trap exit INT

client_touch() {
	file=$1
	for c in $LIVE_CLIENT $FAIL_CLIENTS; do
		echo $DOWN_CLIENTS | grep -q $c && continue
		$PDSH $c touch $TESTDIR/${c}_$file || return 1
	done
}

client_rm() {
	file=$1
	for c in $LIVE_CLIENT $FAIL_CLIENTS; do
		$PDSH $c rm $TESTDIR/${c}_$file
	done
}

client_mkdirs() {
	for c in $LIVE_CLIENT $FAIL_CLIENTS; do
		echo "$c mkdir $TESTDIR/$c"
		$PDSH $c "mkdir $TESTDIR/$c && ls -l $TESTDIR/$c"
	done
}

client_rmdirs() {
	for c in $LIVE_CLIENT $FAIL_CLIENTS; do
		echo "rmdir $TESTDIR/$c"
		$PDSH $LIVE_CLIENT "rmdir $TESTDIR/$c"
	done
}

clients_recover_osts() {
    facet=$1
#    do_node $CLIENTS "$LCTL "'--device %OSC_`hostname`_'"${facet}_svc_MNT_client_facet recover"
}

check_and_setup_lustre

rm -rf $TESTDIR
mkdir -p $TESTDIR

test_0() {
	for i in $(seq $MDSCOUNT) ; do
		fail mds$i
	done

	for i in $(seq $OSTCOUNT) ; do
		fail ost$i
	done
	return 0
}
run_test 0 "Fail all nodes, independently"

############### First Failure Mode ###############
test_1() {
	[ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs"

	clients_up

	shutdown_facet mds1
	reboot_facet mds1

	# prepare for MDS failover
	change_active mds1
	reboot_facet mds1

	clients_up &
	DFPID=$!
	sleep 5

	shutdown_facet mds2

	echo "Reintegrating MDS2"
	reboot_facet mds2
	wait_for_facet mds2
	start_mdt 2 || return 2

	wait_for_facet mds1
	start_mdt 1 || return $?

	#Check FS
	wait $DFPID
	echo "Verify reintegration"
	clients_up || return 1
}
run_test 1 "MDS/MDS failure"
###################################################

############### Second Failure Mode ###############
test_2() {
	echo "Verify Lustre filesystem is up and running"
	[ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"

	clients_up

	for i in $(seq $MDSCOUNT) ; do
		shutdown_facet mds$i
		reboot_facet mds$i

		# prepare for MDS failover
		change_active mds$i
		reboot_facet mds$i
	done

	clients_up &
	DFPID=$!
	sleep 5

	shutdown_facet ost1

	echo "Reintegrating OST"
	reboot_facet ost1
	wait_for_facet ost1
	start_ost 1 || return 2

	for i in $(seq $MDSCOUNT) ; do
		wait_for_facet mds$i
		start_mdt $i || return $?
	done

	#Check FS
	wait $DFPID
	clients_recover_osts ost1
	echo "Verify reintegration"
	clients_up || return 1

}
run_test 2 "Second Failure Mode: MDS/OST `date`"
###################################################

############### Third Failure Mode ###############
test_3() {
	#Create files
	echo "Verify Lustre filesystem is up and running"
	[ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"

	#MDS Portion
	for i in $(seq $MDSCOUNT) ; do
		fail mds$i
	done

	#Check FS
	echo "Test Lustre stability after MDS failover"
	clients_up

	#CLIENT Portion
	echo "Failing 2 CLIENTS"
	fail_clients 2

	#Check FS
	echo "Test Lustre stability after CLIENT failure"
	clients_up

	#Reintegration
	echo "Reintegrating CLIENTS"
	reintegrate_clients || return 1

	clients_up || return 3
	sleep 2 # give it a little time for fully recovered before next test
}
run_test 3  "Third Failure Mode: MDS/CLIENT `date`"
###################################################

############### Fourth Failure Mode ###############
test_4() {
	echo "Fourth Failure Mode: OST/MDS `date`"

	#OST Portion
	shutdown_facet ost1

	#Check FS
	echo "Test Lustre stability after OST failure"
	clients_up &
	DFPIDA=$!
	sleep 5

	for i in $(seq $MDSCOUNT) ; do
		shutdown_facet mds$i
		reboot_facet mds$i

		# prepare for MDS failover
		change_active mds$i
		reboot_facet mds$i
	done

	clients_up &
	DFPIDB=$!
	sleep 5

	#Reintegration
	echo "Reintegrating OST"
	reboot_facet ost1
	wait_for_facet ost1
	start_ost 1

	for i in $(seq $MDSCOUNT) ; do
		wait_for_facet mds$i
		start_mdt $i || return $?
	done

	#Check FS
	wait $DFPIDA
	wait $DFPIDB
	clients_recover_osts ost1
	echo "Test Lustre stability after MDS failover"
	clients_up || return 1
}
run_test 4 "Fourth Failure Mode: OST/MDS `date`"
###################################################

############### Fifth Failure Mode ###############
test_5() {
	[ $OSTCOUNT -lt 2 ] && skip_env "needs >= 2 OSTs"

	echo "Fifth Failure Mode: OST/OST `date`"

	#Create files
	echo "Verify Lustre filesystem is up and running"
	[ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"

	clients_up
	#OST Portion
	shutdown_facet ost1
	reboot_facet ost1

	#Check FS
	echo "Test Lustre stability after OST failure"
	clients_up &
	DFPIDA=$!
	sleep 5

	#OST Portion
	shutdown_facet ost2
	reboot_facet ost2

	#Check FS
	echo "Test Lustre stability after OST failure"
	clients_up &
	DFPIDB=$!
	sleep 5

	#Reintegration
	echo "Reintegrating OSTs"
	wait_for_facet ost1
	start_ost 1
	wait_for_facet ost2
	start_ost 2

	clients_recover_osts ost1
	clients_recover_osts ost2
	sleep $TIMEOUT

	wait $DFPIDA
	wait $DFPIDB
	clients_up || return 2
}
run_test 5 "Fifth Failure Mode: OST/OST `date`"
###################################################

############### Sixth Failure Mode ###############
test_6() {
	echo "Sixth Failure Mode: OST/CLIENT `date`"

	#Create files
	echo "Verify Lustre filesystem is up and running"
	[ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"

	clients_up
	client_touch testfile || return 2

	#OST Portion
	shutdown_facet ost1
	reboot_facet ost1

	#Check FS
	echo "Test Lustre stability after OST failure"
	clients_up &
	DFPIDA=$!
	echo DFPIDA=$DFPIDA
	sleep 5

	#CLIENT Portion
	echo "Failing CLIENTs"
	fail_clients

	#Check FS
	echo "Test Lustre stability after CLIENTs failure"
	clients_up &
	DFPIDB=$!
	echo DFPIDB=$DFPIDB
	sleep 5

	#Reintegration
	echo "Reintegrating OST/CLIENTs"
	wait_for_facet ost1
	start_ost 1
	reintegrate_clients || return 1
	sleep 5

	wait_remote_prog "stat -f" $((TIMEOUT * 3 + 20))
	wait $DFPIDA
	wait $DFPIDB

	echo "Verifying mount"
	[ -z "$(mounted_lustre_filesystems)" ] && return 3
	clients_up
}
run_test 6 "Sixth Failure Mode: OST/CLIENT `date`"
###################################################

############### Seventh Failure Mode ###############
test_7() {
	echo "Seventh Failure Mode: CLIENT/MDS `date`"

	#Create files
	echo "Verify Lustre filesystem is up and running"
	[ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"

	clients_up
	client_touch testfile  || return 1

	#CLIENT Portion
	echo "Part 1: Failing CLIENT"
	fail_clients 2

	#Check FS
	echo "Test Lustre stability after CLIENTs failure"
	clients_up
	$PDSH $LIVE_CLIENT "ls -l $TESTDIR"
	$PDSH $LIVE_CLIENT "rm -f $TESTDIR/*_testfile"

	#Sleep
	echo "Wait 1 minutes"
	sleep 60

	#Create files
	echo "Verify Lustre filesystem is up and running"
	[ -z "$(mounted_lustre_filesystems)" ] && return 2

	clients_up
	client_rm testfile

	#MDS Portion
	for i in $(seq $MDSCOUNT) ; do
		fail mds$i
	done

	$PDSH $LIVE_CLIENT "ls -l $TESTDIR"
	$PDSH $LIVE_CLIENT "rm -f $TESTDIR/*_testfile"

	#Reintegration
	echo "Reintegrating CLIENTs"
	reintegrate_clients || return 2
	clients_up

	#Sleep
	echo "wait 1 minutes"
	sleep 60
}
run_test 7 "Seventh Failure Mode: CLIENT/MDS `date`"
###################################################

############### Eighth Failure Mode ###############
test_8() {
	echo "Eighth Failure Mode: CLIENT/OST `date`"

	#Create files
	echo "Verify Lustre filesystem is up and running"
	[ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"

	clients_up
	client_touch testfile

	#CLIENT Portion
	echo "Failing CLIENTs"
	fail_clients 2

	#Check FS
	echo "Test Lustre stability after CLIENTs failure"
	clients_up
	$PDSH $LIVE_CLIENT "ls -l $TESTDIR"
	$PDSH $LIVE_CLIENT "rm -f $TESTDIR/*_testfile"

	#Sleep
	echo "Wait 1 minutes"
	sleep 60

	#Create files
	echo "Verify Lustre filesystem is up and running"
	[ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"

	clients_up
	client_touch testfile

	#OST Portion
	shutdown_facet ost1
	reboot_facet ost1

	#Check FS
	echo "Test Lustre stability after OST failure"
	clients_up &
	DFPID=$!
	sleep 5
	#non-failout hangs forever here
	#$PDSH $LIVE_CLIENT "ls -l $TESTDIR"
	#$PDSH $LIVE_CLIENT "rm -f $TESTDIR/*_testfile"

	#Reintegration
	echo "Reintegrating CLIENTs/OST"
	reintegrate_clients || return 3
	wait_for_facet ost1
	start_ost 1
	wait $DFPID
	clients_up || return 1
	client_touch testfile2 || return 2

	#Sleep
	echo "Wait 1 minutes"
	sleep 60
}
run_test 8 "Eighth Failure Mode: CLIENT/OST `date`"
###################################################

############### Ninth Failure Mode ###############
test_9() {
	#Create files
	echo "Verify Lustre filesystem is up and running"
	[ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"

	clients_up
	client_touch testfile || return 1

	#CLIENT Portion
	echo "Failing CLIENTs"
	fail_clients 2

	#Check FS
	echo "Test Lustre stability after CLIENTs failure"
	clients_up
	$PDSH $LIVE_CLIENT "ls -l $TESTDIR" || return 1
	$PDSH $LIVE_CLIENT "rm -f $TESTDIR/*_testfile" || return 2

	#Sleep
	echo "Wait 1 minutes"
	sleep 60

	#Create files
	echo "Verify Lustre filesystem is up and running"
	client_up $LIVE_CLIENT || return 3
	client_touch testfile || return 4

	#CLIENT Portion
	echo "Failing CLIENTs"
	fail_clients 2

	#Check FS
	echo "Test Lustre stability after CLIENTs failure"
	clients_up
	$PDSH $LIVE_CLIENT "ls -l $TESTDIR" || return 5
	$PDSH $LIVE_CLIENT "rm -f $TESTDIR/*_testfile" || return 6

	#Reintegration
	echo "Reintegrating  CLIENTs/CLIENTs"
	reintegrate_clients || return 7
	clients_up

	#Sleep
	echo "Wait 1 minutes"
	sleep 60
}
run_test 9 "Ninth Failure Mode: CLIENT/CLIENT `date`"
###################################################

############### Tenth Failure Mode ###############
test_10() {
	[ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs"

	shutdown_facet mds1
	reboot_facet mds1

	# prepare for MDS failover
	change_active mds1
	reboot_facet mds1

	clients_up &
	DFPID=$!
	sleep 5

	shutdown_facet ost1

	echo "Reintegrating OST"
	reboot_facet ost1
	wait_for_facet ost1
	start_ost 1 || return 2

	shutdown_facet mds2
	reboot_facet mds2

	# prepare for MDS failover
	change_active mds2
	reboot_facet mds2

	wait_for_facet mds1
	start_mdt 1 || return $?

	wait_for_facet mds2
	start_mdt 2 || return $?

	#Check FS
	wait $DFPID
	clients_recover_osts ost1
	echo "Verify reintegration"
	clients_up || return 1
}
run_test 10 "Tenth Failure Mode: MDT0/OST/MDT1 `date`"
###################################################

############### Seventh Failure Mode ###############
test_11() {
	[ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs"
	echo "Verify Lustre filesystem is up and running"
	[ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"

	#MDS Portion
	fail mds1
	#Check FS

	echo "Test Lustre stability after MDS failover"
	clients_up

	#CLIENT Portion
	echo "Failing 2 CLIENTS"
	fail_clients 2

	#Check FS
	echo "Test Lustre stability after CLIENT failure"
	clients_up

	#Reintegration
	echo "Reintegrating CLIENTS"
	reintegrate_clients || return 1

	fail mds2

	clients_up || return 3
	sleep 2 # give it a little time for fully recovered before next test
}
run_test 11 "Eleventh Failure Mode: MDS0/CLIENT/MDS1 `date`"
###################################################

test_12() {
	[ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs"
	echo "Verify Lustre filesystem is up and running"
	[ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"

	#MDS Portion
	fail mds1,mds2
	clients_up

	#OSS Portion
	fail ost1,ost2
	clients_up

	#CLIENT Portion
	echo "Failing 2 CLIENTS"
	fail_clients 2

	#Check FS
	echo "Test Lustre stability after CLIENT failure"
	clients_up

	#Reintegration
	echo "Reintegrating CLIENTS"
	reintegrate_clients || return 1

	clients_up || return 3
	sleep 2 # give it a little time for fully recovered before next test
}
run_test 12 "Twelve Failure Mode: MDS0,MDS1/OST0, OST1/CLIENTS `date`"
###################################################

test_13() {
	[ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs"
	echo "Verify Lustre filesystem is up and running"
	[ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"

	#MDS Portion
	fail mds1,mds2
	clients_up

	#CLIENT Portion
	echo "Failing 2 CLIENTS"
	fail_clients 2

	#Check FS
	echo "Test Lustre stability after CLIENT failure"
	clients_up

	#Reintegration
	echo "Reintegrating CLIENTS"
	reintegrate_clients || return 1

	clients_up || return 3
	sleep 2 # give it a little time for fully recovered before next test

	#OSS Portion
	fail ost1,ost2
	clients_up || return 4
}
run_test 13 "Thirteen Failure Mode: MDS0,MDS1/CLIENTS/OST0,OST1 `date`"
###################################################

test_14() {
	[ $MDSCOUNT -lt 2 ] && skip "needs >= 2 MDTs"
	echo "Verify Lustre filesystem is up and running"
	[ -z "$(mounted_lustre_filesystems)" ] && error "Lustre is not running"

	#OST Portion
	fail ost1,ost2
	clients_up

	#CLIENT Portion
	echo "Failing 2 CLIENTS"
	fail_clients 2

	#Check FS
	echo "Test Lustre stability after CLIENT failure"
	clients_up

	#Reintegration
	echo "Reintegrating CLIENTS"
	reintegrate_clients || return 1

	clients_up || return 3
	sleep 2 # give it a little time for fully recovered before next test

	#OSS Portion
	fail mds1,mds2
	clients_up || return 4
}
run_test 14 "Fourteen Failure Mode: OST0,OST1/CLIENTS/MDS0,MDS1 `date`"

complete $SECONDS
check_and_cleanup_lustre
exit_status


#!/bin/bash
#
# krb5_login.sh - obtain and cache Kerberos ticket-granting ticket
#
###############################################################################

#
# nothing need for root
#
[ $UID -eq 0 ] && exit 0

#
# list Kerberos 5 credentials silently
# exit status:
# 0 - klist finds a credentials cache
# 1 - klist does not find a credentials cache or the tickets are expired
#
klist -5 -s && exit 0

# get the user name for uid $UID
GSS_USER=$(getent passwd $UID | cut -d: -f1)

GSS_PASS=${GSS_PASS:-"$GSS_USER"}

echo "***** refresh Kerberos V5 TGT for uid $UID *****"
if [ -z "$GSS_PASS" ]; then
    kinit
else
    expect <<EOF
set timeout 30

log_user 1

set spawnid [spawn /bin/bash]
send "export PS1=\"user@host $ \" \r"
expect {
    timeout {puts "timeout" ;exit 1}
    "user@host $ "
}

send "kinit\r"
expect {
    timeout {puts "timeout" ;exit 1}
    "Password for "
}

send "$GSS_PASS\r"
expect {
    timeout {puts "timeout" ;exit 1}
    "user@host $ "
}

exit 0
EOF
fi
ret=$?
exit $ret


#!/bin/bash
#
# This script is used to test large size LUN support in Lustre.
#
################################################################################
set -e

LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
. $LUSTRE/tests/test-framework.sh
init_test_env $@
. ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
init_logging

if [ "$REFORMAT" != "yes" ]; then
	skip_env "$0 reformats all devices,\
		please set REFORMAT to run this test"
fi

# Variable to run mdsrate
THREADS_PER_CLIENT=${THREADS_PER_CLIENT:-5}    # thread(s) per client node
MACHINEFILE=${MACHINEFILE:-$TMP/$TESTSUITE.machines}
NODES_TO_USE=${NODES_TO_USE:-$CLIENTS}
NUM_CLIENTS=$(get_node_count ${NODES_TO_USE//,/ })

# bug number:
ALWAYS_EXCEPT="$LARGE_LUN_EXCEPT"

build_test_filter
LARGE_LUN_RESTORE_MOUNT=false
if is_mounted $MOUNT || is_mounted $MOUNT2; then
	LARGE_LUN_RESTORE_MOUNT=true
fi
# Unmount and cleanup the Lustre filesystem
cleanupall
load_modules

FULL_MODE=${FULL_MODE:-false}
RUN_FSCK=${RUN_FSCK:-true}
# if SLOW=yes, enable the FULL_MODE
[[ $SLOW = yes ]] && FULL_MODE=true
#########################################################################
# Dump the super block information for the filesystem present on device.
run_dumpfs() {
	local facet=$1
	local dev=$2
	local cmd

	log "dump the super block information on $facet device $dev"
	local fstype=$(facet_fstype $facet)

	case $fstype in
		ldiskfs )
			cmd="$DUMPE2FS -h $dev" ;;
		zfs )
			cmd="$ZDB -l $(zpool_name $facet)" ;;
		* )
			error "unknown fstype!" ;;
	esac

	do_facet $facet "$cmd"
}

# Report Lustre filesystem disk space usage and inodes usage of each MDT/OST.
client_df() {
	local mnt_pnt=$1
	local cmd

	cmd="df -h"
	echo -e "\n# $cmd"
	eval $cmd

	cmd="lfs df -h $mnt_pnt"
	echo -e "\n# $cmd"
	eval $cmd

	cmd="lfs df -i $mnt_pnt"
	echo -e "\n# $cmd"
	eval $cmd
}

# Cleanup the directories and files created by llverfs utility.
cleanup_dirs() {
	local target=$1
	local mnt=${2:-$MOUNT}
	local cmd="rm -rf $mnt/{llverfs,dir}*"
	do_facet $target "$cmd"
}

# Run mdsrate.
run_mdsrate() {
	generate_machine_file $NODES_TO_USE $MACHINEFILE ||
		error "can not generate machinefile"

	# set the default stripe count for files in this test to one
	local testdir=$MOUNT/mdsrate
	mkdir -p $testdir
	chmod 0777 $testdir
	$LFS setstripe $testdir -i 0 -c 1
	get_stripe $testdir

	local num_dirs=$THREADS_PER_CLIENT
	[[ $num_dirs -eq 0 ]] && num_dirs=1
	local free_inodes=$(lfs df -i $MOUNT | grep "OST:0" | awk '{print $4}')
	local num_files
	num_files=$((free_inodes / num_dirs))

	local command="$MDSRATE $MDSRATE_DEBUG --create --verbose \
		--ndirs $num_dirs --dirfmt '$testdir/dir%d' \
		--nfiles $num_files --filefmt 'file%%d'"

	echo "# $command"
	mpi_run -machinefile $MACHINEFILE \
		-np $((NUM_CLIENTS * THREADS_PER_CLIENT)) $command

	if [ ${PIPESTATUS[0]} != 0 ]; then
		error "mdsrate create failed"
	fi
}

check_fsfacet() {
	local facet=$1
	local fstype=$(facet_fstype $facet)

	case $fstype in
	    ldiskfs)
		run_e2fsck $(facet_active_host $facet) $(facet_device $facet) \
		    "-y" || error "run e2fsck error"
		;;
	    zfs)
		# Could call fsck.zfs, but currently it does nothing,
		# Could also call zpool scrub, but that could take a LONG time
		# do_facet $facet "fsck.zfs $(facet_device $facet)"
		;;
	esac
}

# Run e2fsck on MDS and OST
do_fsck() {
	$RUN_FSCK || return

	check_fsfacet $SINGLEMDS

	for num in $(seq $OSTCOUNT); do
		check_fsfacet ost${num}
	done
}
################################## Main Flow ###################################
trap cleanupall EXIT

test_1 () {
	[ "$mds1_FSTYPE" != ldiskfs ] && skip_env "ldiskfs only test"

	local dev

	for num in $(seq $OSTCOUNT); do
		dev=$(ostdevname $num)
		log "run llverdev on the OST $dev"
		do_rpc_nodes $(facet_host ost${num}) run_llverdev $dev -vpf ||
			error "llverdev on $dev failed!"
	done
	# restore format overwritten by llverdev
	formatall
}
run_test 1 "run llverdev on raw LUN"

test_2 () {
	local dev
	local ostmnt
	local fstype
	local zostsize_restore=${OSTSIZE}
	local zmin=$((30 << 30)) # 30GiB in bytes
	local skipped=1

	stack_trap "export OSTSIZE=$zostsize_restore" EXIT

	for num in $(seq $OSTCOUNT); do
		dev=$(ostdevname $num)
		ostmnt=$(facet_mntpt ost${num})
		fstype=$(facet_fstype ost${num})

		if [[ $fstype == "zfs" ]] && [[ ${OSTSIZE} -lt ${zmin} ]]; then
			local real_dev=$(ostvdevname $num)
			local num_sectors=$(get_num_sectors $facet $real_dev)
			local phy_bytes=$((num_sectors * 512))

			if [ ${phy_bytes} -lt ${zmin} ] ; then
				log "ost${num}: OSTSIZE ${OSTSIZE} less than 30GiB"
				log "ost${num}: Block device ${phy_bytes} too small"
				log " .. skipping this ost"
				continue
			fi
			# Backing block device is big enough
			skipped=0
			log "ost${num}: OSTSIZE ${OSTSIZE} too small, increasing to 30GiB [temporarily]"
			format_ost ${num}
			# NOTE: OSTSIZE is in KB
			export OSTSIZE=$((zmin >> 10))
		fi

		# Mount the OST as an ldiskfs or zfs filesystem.
		log "mount the OST $dev as a $fstype filesystem"
		add ost${num} $(mkfs_opts ost${num} $dev) $FSTYPE_OPT \
			--reformat $(ostdevname $num) \
			$(ostvdevname $num) > /dev/null ||
			error "format ost${num} error"
		if [ $fstype == zfs ]; then
			import_zpool ost${num}
			do_facet ost${num} \
			    "$ZFS set canmount=on $dev; " \
			    "$ZFS set mountpoint=legacy $dev; " \
			    "$ZFS list $dev"
		fi
		run_dumpfs ost${num} $dev
		do_facet ost${num} mount -t $fstype $dev \
			$ostmnt "$OST_MOUNT_OPTS"

		# Run llverfs on the mounted filesystem in partial mode
		# to ensure that the kernel can perform filesystem operations
		# on the complete device without any errors.
		log "run llverfs in partial mode on the OST $fstype $ostmnt"
		do_rpc_nodes $(facet_host ost${num}) run_llverfs $ostmnt -vpl \
			"no" || error "run_llverfs error on $fstype"

		# Unmount the OST.
		log "unmount the OST $dev"
		stop ost${num}

		# After llverfs is run on the filesystem in partial
		# mode, a full e2fsck should be run to catch any errors early.
		$RUN_FSCK && check_fsfacet ost${num}

		if $FULL_MODE; then
			log "full mode, mount the OST $dev as a $fstype again"
			if [ $fstype == zfs ]; then
				import_zpool ost${num}
			fi
			do_facet ost${num} mount -t $(facet_fstype ost${num}) \
				$dev $ostmnt "$OST_MOUNT_OPTS"
			cleanup_dirs ost${num} $ostmnt
			do_facet ost${num} "sync"

			run_dumpfs ost${num} $dev

			# Run llverfs on the mounted ldiskfs filesystem in full
			# mode to ensure that the kernel can perform filesystem
			# operations on the complete device without any errors.
			log "run llverfs in full mode on OST $fstype $ostmnt"
			do_rpc_nodes $(facet_host ost${num}) run_llverfs \
				$ostmnt -vl "no" ||
				error "run_llverfs error on $fstype"

			# Unmount the OST.
			log "unmount the OST $dev"
			stop ost${num}

			# After llverfs is run on the ldiskfs filesystem in
			# full mode, a full e2fsck should be run to catch any
			#  errors early.
			$RUN_FSCK && check_fsfacet ost${num}
		fi
		export OSTSIZE=${zostsize_restore}
	done
	[[ $skipped -ne 0 ]] && skip_env "No OST with enough space is available."
	# there is no reason to continue using ost devices
	# filled by llverfs as ldiskfs
	formatall
}
run_test 2 "run llverfs on OST ldiskfs/zfs filesystem"

test_3 () {
	[ -z "$CLIENTS" ] && skip_env "CLIENTS not defined, skipping"
	[ -z "$MPIRUN" ] && skip_env "MIPRUN not defined, skipping"
	[ -z "$MDSRATE" ] && skip_env "MDSRATE not defined, skipping"
	[ ! -x $MDSRATE ] && skip_env "$MDSRATE not built, skipping"
	# Setup the Lustre filesystem.
	log "setup the lustre filesystem"
	REFORMAT="yes" check_and_setup_lustre

	log "run mdsrate to use up the free inodes."
	# Run the mdsrate test suite.
	run_mdsrate
	client_df $MOUNT

	sync; sleep 5; sync
	stopall
	do_fsck
}
run_test 3 "use up free inodes on the OST with mdsrate"

test_4 () {
	# Setup the Lustre filesystem.
	log "setup the lustre filesystem"
	REFORMAT="yes" check_and_setup_lustre
	local dev

	for num in $(seq $OSTCOUNT); do
		dev=$(ostdevname $num)
		run_dumpfs ost${num} $dev
	done

	# Run llverfs on the mounted Lustre filesystem both in partial and
	# full mode to to fill the filesystem and verify the file contents.
	log "run llverfs in partial mode on the Lustre filesystem $MOUNT"
	run_llverfs $MOUNT -vp "no" || error "run_llverfs error on lustre"
	client_df $MOUNT

	sync; sleep 5; sync
	stopall
	do_fsck

	if $FULL_MODE; then
		# Setup the Lustre filesystem again.
		log "setup the lustre filesystem again"
		setupall

		cleanup_dirs client $MOUNT
		sync
		client_df $MOUNT

		for num in $(seq $OSTCOUNT); do
			dev=$(ostdevname $num)
			run_dumpfs ost${num} $dev
		done

		log "run llverfs in full mode on the Lustre filesystem $MOUNT"
		run_llverfs $MOUNT -vl "no" ||
			error "run_llverfs error on lustre"
		client_df $MOUNT

		sync; sleep 5; sync
		stopall
		do_fsck
	fi
}
run_test 4 "run llverfs on lustre filesystem"

complete $SECONDS
$LARGE_LUN_RESTORE_MOUNT && setupall
check_and_cleanup_lustre
exit_status


#!/bin/bash

set -e

PTLDEBUG=${PTLDEBUG:--1}
SETUP=${SETUP:-""}
CLEANUP=${CLEANUP:-""}

LUSTRE=${LUSTRE:-$(dirname $0)/..}
. $LUSTRE/tests/test-framework.sh
init_test_env $@
init_logging

ALWAYS_EXCEPT="$LARGE_SCALE_EXCEPT "

build_test_filter

remote_mds_nodsh && skip "remote MDS with nodsh"

[ -z "$CLIENTS" ] && skip_env "$TESTSUITE: Need two or more clients"
[ $CLIENTCOUNT -lt 2 ] &&
	skip_env "$TESTSUITE: Need 2+ clients, have only $CLIENTCOUNT"

MOUNT_2=""

check_and_setup_lustre
rm -rf $DIR/[df][0-9]*

get_mpiuser_id $MPI_USER
MPI_RUNAS=${MPI_RUNAS:-"runas -u $MPI_USER_UID -g $MPI_USER_GID"}
$GSS_KRB5 && refresh_krb5_tgt $MPI_USER_UID $MPI_USER_GID $MPI_RUNAS

[ "$DAEMONFILE" ] && $LCTL debug_daemon start $DAEMONFILE $DAEMONSIZE

test_3a() {
	assert_env CLIENTS MDSRATE MPIRUN

	local -a nodes=(${CLIENTS//,/ })
	# INCREMENT is a number of clients a half of clients by default
	local increment=${INCREMENT:-$(( CLIENTCOUNT / 2 ))}
	local num=$increment
	local LOG=$TMP/${TESTSUITE}_$tfile
	local var=${SINGLEMDS}_svc
	local procfile="*.${!var}.recovery_status"
	local iters=${ITERS:-3}
	local nfiles=${NFILES:-50000}
	local nthreads=${THREADS_PER_CLIENT:-3}
	local IFree=$(inodes_available)
	local pid
	local list
	local -a res
	local dir=$DIR/d0.$TESTNAME

	[ $IFree -gt $nfiles ] || nfiles=$IFree

	mkdir -p $dir
	chmod 0777 $dir

	while [ $num -le $CLIENTCOUNT ]; do
		list=$(comma_list ${nodes[@]:0:$num})

		generate_machine_file $list $MACHINEFILE ||
			error "can not generate machinefile"

		for i in $(seq $iters); do
			mdsrate_cleanup $num $MACHINEFILE $nfiles $dir 'f%%d' \
				--ignore

			COMMAND="${MDSRATE} --create --nfiles $nfiles --dir
				 $dir --filefmt 'f%%d'"
			mpi_run ${MACHINEFILE_OPTION} $MACHINEFILE \
				-np $((num * nthreads)) ${COMMAND} | tee ${LOG}&

			pid=$!
			echo "pid=$pid"

			# 2 threads 100000 creates 117 secs
			sleep 20

			log "$i : Starting failover on $SINGLEMDS"
			facet_failover $SINGLEMDS
			if ! wait_recovery_complete $SINGLEMDS \
			     $((TIMEOUT * 10)); then
				echo "$SINGLEMDS recovery is not completed!"
				kill -9 $pid
				exit 7
			fi

			duration=$(do_facet $SINGLEMDS lctl get_param -n \
				$procfile | grep recovery_duration)

			res=( "${res[@]}" "$num" )
			res=( "${res[@]}" "$duration" )
			echo "RECOVERY TIME: NFILES=$nfiles number of clients: $num $duration"
			wait $pid
		done
		num=$((num + increment))
	done

	mdsrate_cleanup $num $MACHINEFILE $nfiles $dir 'f%%d' --ignore

	i=0
	while [ $i -lt ${#res[@]} ]; do
		echo "RECOVERY TIME: NFILES=$nfiles number of clients: ${res[i]}  ${res[i+1]}"
		i=$((i+2))
	done
}

run_test 3a "recovery time, $CLIENTCOUNT clients"

complete $SECONDS
check_and_cleanup_lustre
exit_status


#!/bin/bash

set -e

ONLY=${ONLY:-"$*"}
ALWAYS_EXCEPT="$LFSCK_PERFORMANCE_EXCEPT"
[ "$SLOW" = "no" ] && EXCEPT_SLOW=""
# UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT!

LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)}
. $LUSTRE/tests/test-framework.sh
init_test_env $@
. ${CONFIG:=$LUSTRE/tests/cfg/$NAME.sh}
init_logging

require_dsh_mds || exit 0
require_dsh_ost || exit 0

[ "$SLOW" = "no" ] &&
	skip "skip lfsck performance test under non-SLOW mode"

NTHREADS=${NTHREADS:-0}
UNIT=${UNIT:-8192}
MINCOUNT=${MINCOUNT:-4096}
MAXCOUNT=${MAXCOUNT:-8192}
MINCOUNT_REPAIR=${MINCOUNT_REPAIR:-4096}
MAXCOUNT_REPAIR=${MAXCOUNT_REPAIR:-8192}
BASE_COUNT=${BASE_COUNT:-8192}
FACTOR=${FACTOR:-2}
INCFACTOR=${INCFACTOR:-25} #percent
MINSUBDIR=${MINSUBDIR:-1}
MAXSUBDIR=${MAXSUBDIR:-2}
TOTSUBDIR=${TOTSUBDIR:-2}
WTIME=${WTIME:-86400}

RCMD="do_facet ${SINGLEMDS}"
RLCTL="${RCMD} ${LCTL}"
MDT_DEV="${FSNAME}-MDT0000"
MDT_DEVNAME=$(mdsdevname ${SINGLEMDS//mds/})
START_NAMESPACE="${RLCTL} lfsck_start -M ${MDT_DEV} -t namespace"
STOP_LFSCK="${RLCTL} lfsck_stop -M ${MDT_DEV} -A"
SHOW_NAMESPACE="${RLCTL} get_param -n mdd.${MDT_DEV}.lfsck_namespace"
MNTOPTS_NOSCRUB="-o user_xattr,noscrub"
remote_mds && ECHOCMD=${RCMD} || ECHOCMD="eval"

if [ ${NTHREADS} -eq 0 ]; then
	CPUCORE=$(${RCMD} cat /proc/cpuinfo | grep "processor.*:" | wc -l)
	NTHREADS=$((CPUCORE * 2))
fi

lfsck_attach() {
	${RCMD} "modprobe obdecho"

	${ECHOCMD} "${LCTL} <<-EOF
		attach echo_client lfsck-MDT0000 lfsck-MDT0000_UUID
		setup ${MDT_DEV} mdd
	EOF"
}

lfsck_detach() {
	${ECHOCMD} "${LCTL} <<-EOF
		device lfsck-MDT0000
		cleanup
		detach
	EOF"
}

lfsck_create() {
	local echodev=$(${RLCTL} dl | grep echo_client|awk '{print $1}')
	local j

	${ECHOCMD} "${LCTL} <<-EOF
		cfg_device ${echodev}
		test_mkdir ${tdir}
	EOF"

	for ((j = 1; j < ${threads}; j++)); do
		${ECHOCMD} "${LCTL} <<-EOF
			cfg_device ${echodev}
			test_mkdir ${tdir}${j}
		EOF"
	done

	${ECHOCMD} "${LCTL} <<-EOF
		cfg_device ${echodev}
		--threads ${threads} 0 ${echodev} test_create \
		-d ${tdir} -D ${threads} -b ${lbase} -c 0 -n ${usize}
	EOF"
}

lfsck_cleanup() {
	do_rpc_nodes $(facet_active_host $SINGLEMDS) unload_modules
	REFORMAT="yes" cleanup_and_setup_lustre
}

lfsck_create_nfiles() {
	local total=$1
	local lbase=$2
	local threads=$3
	local linkea=$4
	local ldir="/test-${lbase}"
	local cycle=0
	local count=${UNIT}

	while true; do
		[ ${count} -eq 0 -o  ${count} -gt ${total} ] && count=${total}
		local usize=$((count / NTHREADS))
		[ ${usize} -eq 0 ] && break
		local tdir=${ldir}-${cycle}-

		echo "[cycle: ${cycle}] [threads: ${threads}]"\
		     "[files: ${count}] [basedir: ${tdir}]"
		start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB ||
			error "Fail to start MDS!"
		#define OBD_FAIL_FID_IGIF	0x1504
		[ ! -z $linkea ] && ${RLCTL} set_param fail_loc=0x1504

		lfsck_attach
		lfsck_create
		lfsck_detach

		[ ! -z $linkea ] && ${RLCTL} set_param fail_loc=0x0
		stop ${SINGLEMDS} || error "Fail to stop MDS!"

		total=$((total - usize * NTHREADS))
		[ ${total} -eq 0 ] && break
		lbase=$((lbase + usize))
		cycle=$((cycle + 1))
	done
}

build_test_filter

format_start_mgs () {
	do_rpc_nodes $(facet_active_host mgs) load_modules_local
	format_mgs
	start mgs $(mgsdevname) $MGS_MOUNT_OPTS
}

test_0() {
	local BCOUNT=0
	local i

	stopall
	combined_mgs_mds || format_start_mgs
	do_rpc_nodes $(facet_active_host $SINGLEMDS) load_modules_local
	format_mdt $(facet_number $SINGLEMDS)

	for ((i = $MINCOUNT; i <= $MAXCOUNT; i = $((i * FACTOR)))); do
		local nfiles=$((i - BCOUNT))

		echo "+++ start to create for ${i} files set at: $(date) +++"
		lfsck_create_nfiles ${nfiles} ${BCOUNT} ${NTHREADS} ||
			error "Fail to create files!"
		echo "+++ end to create for ${i} files set at: $(date) +++"

		BCOUNT=${i}
		start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB > /dev/null ||
			error "Fail to start MDS!"

		echo "start lfsck_namespace for ${i} files set at: $(date)"
		$START_NAMESPACE || error "Fail to start lfsck_namespace!"

		while true; do
			local STATUS=$($SHOW_NAMESPACE |
					awk '/^status/ { print $2 }')
			[ "$STATUS" == "completed" ] && break
			sleep 3 # check status every 3 seconds
		done

		echo "end lfsck_namespace for ${i} files set at: $(date)"
		SPEED=$($SHOW_NAMESPACE |
			awk '/^average_speed_phase1/ { print $2 }')
		echo "lfsck_namespace speed is ${SPEED}/sec"
		stop ${SINGLEMDS} > /dev/null || error "Fail to stop MDS!"
	done
	combined_mgs_mds || stop mgs
}
run_test 0 "lfsck namespace performance (routine case) without load"

test_1() {
	[ "$mds1_FSTYPE" != ldiskfs ] &&
		skip_env "not implemented for ZFS"

	local BCOUNT=0
	local i

	stopall
	combined_mgs_mds || format_start_mgs
	do_rpc_nodes $(facet_active_host $SINGLEMDS) load_modules_local
	format_mdt $(facet_number $SINGLEMDS)

	for ((i = $MINCOUNT_REPAIR; i <= $MAXCOUNT_REPAIR;
	      i = $((i * FACTOR)))); do
		local nfiles=$((i - BCOUNT))

		echo "+++ start to create for ${i} files set at: $(date) +++"
		lfsck_create_nfiles ${nfiles} ${BCOUNT} ${NTHREADS} ||
			error "Fail to create files!"
		echo "+++ end to create for ${i} files set at: $(date) +++"

		BCOUNT=${i}
		local stime=$(date +%s)
		echo "backup/restore ${i} files start at: $(date)"
		mds_backup_restore $SINGLEMDS || error "Fail to backup/restore!"
		echo "backup/restore ${i} files end at: $(date)"
		local etime=$(date +%s)
		local delta=$((etime - stime))
		[ $delta -gt 0 ] || delta=1
		echo "backup/restore ${i} files used ${delta} seconds"
		echo "backup/restore speed is $((i / delta))/sec"

		start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB > /dev/null ||
			error "Fail to start MDS!"

		echo "start lfsck_namespace for ${i} files set at: $(date)"
		$START_NAMESPACE || error "Fail to start lfsck_namespace!"

		while true; do
			local STATUS=$($SHOW_NAMESPACE |
					awk '/^status/ { print $2 }')
			[ "$STATUS" == "completed" ] && break
			sleep 3 # check status every 3 seconds
		done

		echo "end lfsck_namespace for ${i} files set at: $(date)"
		local SPEED=$($SHOW_NAMESPACE |
			      awk '/^average_speed_phase1/ { print $2 }')
		echo "lfsck_namespace speed is ${SPEED}/sec"
		stop ${SINGLEMDS} > /dev/null || error "Fail to stop MDS!"
	done
	combined_mgs_mds || stop mgs
}
run_test 1 "lfsck namespace performance (backup/restore) without load"

test_2() {
	local i

	for ((i = $MINCOUNT_REPAIR; i <= $MAXCOUNT_REPAIR;
	      i = $((i * FACTOR)))); do
		stopall
		combined_mgs_mds || format_start_mgs
		do_rpc_nodes $(facet_active_host $SINGLEMDS) load_modules_local
		format_mdt $(facet_number $SINGLEMDS)

		echo "+++ start to create for ${i} files set at: $(date) +++"
		lfsck_create_nfiles ${i} 0 ${NTHREADS} 1 ||
			error "Fail to create files!"
		echo "+++ end to create for ${i} files set at: $(date) +++"

		start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB > /dev/null ||
			error "Fail to start MDS!"

		echo "start lfsck_namespace for ${i} files set at: $(date)"
		$START_NAMESPACE || error "Fail to start lfsck_namespace!"

		while true; do
			local STATUS=$($SHOW_NAMESPACE |
					awk '/^status/ { print $2 }')
			[ "$STATUS" == "completed" ] && break
			sleep 3 # check status every 3 seconds
		done

		echo "end lfsck_namespace for ${i} files set at: $(date)"
		local SPEED=$($SHOW_NAMESPACE |
			      awk '/^average_speed_phase1/ { print $2 }')
		echo "lfsck_namespace speed is ${SPEED}/sec"
		stop ${SINGLEMDS} > /dev/null || error "Fail to stop MDS!"
	done
	combined_mgs_mds || stop mgs
}
run_test 2 "lfsck namespace performance (upgrade from 1.8) without load"

test_3() {
	[ $INCFACTOR -gt 25 ] && INCFACTOR=25

	local inc_count=$((BASE_COUNT * INCFACTOR / 100))
	local BCOUNT=0
	local i

	stopall
	combined_mgs_mds || format_start_mgs
	do_rpc_nodes $(facet_active_host $SINGLEMDS) load_modules_local
	format_mdt $(facet_number $SINGLEMDS)

	for ((i = $inc_count; i <= $BASE_COUNT; i = $((i + inc_count)))); do
		local nfiles=$((i - BCOUNT))

		echo "+++ start to create for ${i} files set at: $(date) +++"
		lfsck_create_nfiles ${nfiles} ${BCOUNT} ${NTHREADS} ||
			error "Fail to create files!"
		echo "+++ end to create for ${i} files set at: $(date) +++"
		BCOUNT=${i}
	done

	start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB > /dev/null ||
		error "Fail to start MDS!"

	echo "start lfsck_namespace for ${BASE_COUNT} files set at: $(date)"
	$START_NAMESPACE || error "Fail to start lfsck_namespace!"

	while true; do
		local STATUS=$($SHOW_NAMESPACE |
				awk '/^status/ { print $2 }')
		[ "$STATUS" == "completed" ] && break
		sleep 3 # check status every 3 seconds
	done

	echo "end lfsck_namespace for ${BASE_COUNT} files set at: $(date)"
	local FULL_SPEED=$($SHOW_NAMESPACE |
		      awk '/^average_speed_phase1/ { print $2 }')
	echo "lfsck_namespace full_speed is ${FULL_SPEED}/sec"
	stop ${SINGLEMDS} > /dev/null || error "Fail to stop MDS!"
	local inc_speed=$((FULL_SPEED * INCFACTOR / 100))
	local j

	for ((j = $inc_speed; j < $FULL_SPEED; j = $((j + inc_speed)))); do
		start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB > /dev/null ||
			error "Fail to start MDS!"

		$STOP_LFSCK > /dev/null 2>&1
		echo "start lfsck_namespace with speed ${j} at: $(date)"
		$START_NAMESPACE --reset -s ${j} ||
			error "Fail to start lfsck_namespace with speed ${j}!"
		# lfsck_namespace will be paused when MDS stop,
		# and will be restarted automatically when mount up again.
		stop ${SINGLEMDS} > /dev/null || error "Fail to stop MDS!"

		local nfiles=$(((i - BCOUNT) / 2))

		echo "+++ start to create for ${i} files set at: $(date) +++"
		lfsck_create_nfiles ${nfiles} ${BCOUNT} ${NTHREADS} ||
			error "Fail to create files!"
		echo "+++ end to create for ${i} files set at: $(date) +++"
		BCOUNT=${i}
		i=$((i + inc_count))
	done

	start ${SINGLEMDS} $MDT_DEVNAME $MNTOPTS_NOSCRUB > /dev/null ||
		error "Fail to start MDS!"

	$STOP_LFSCK /dev/null 2>&1
	echo "start lfsck_namespace with full speed at: $(date)"
	$START_NAMESPACE --reset -s 0 ||
		error "Fail to start lfsck_namespace with full speed!"
	stop ${SINGLEMDS} > /dev/null || error "Fail to stop MDS!"

	local nfiles=$(((i - BCOUNT) / 2))

	echo "+++ start to create for ${i} files set at: $(date) +++"
	lfsck_create_nfiles ${nfiles} ${BCOUNT} ${NTHREADS} ||
		error "Fail to create files!"
	echo "+++ end to create for ${i} files set at: $(date) +++"
	combined_mgs_mds || stop mgs
}
run_test 3 "lfsck namespace impact on create performance"

show_layout() {
	local idx=$1

	do_facet mds${idx} \
		"$LCTL get_param -n mdd.$(facet_svc mds${idx}).lfsck_layout"
}

layout_test_one() {
	echo "***** Start layout LFSCK on all devices at: $(date) *****"
	$RLCTL lfsck_start -M ${MDT_DEV} -t layout -A -r || return 21

	wait_update_facet $SINGLEMDS "$LCTL get_param -n \
		mdd.${MDT_DEV}.lfsck_layout |
		awk '/^status/ { print \\\$2 }'" "completed" $WTIME || {
		show_layout 1
		return 22
	}
	echo "***** End layout LFSCK on all devices at: $(date) *****"

	for n in $(seq $MDSCOUNT); do
		show_layout ${n}

		local SPEED=$(show_layout ${n} |
			      awk '/^average_speed_phase1/ { print $2 }')
		echo
		echo "lfsck_layout speed on MDS_${n} is $SPEED objs/sec"
		echo
	done
}

layout_gen_one() {
	local idx1=$1
	local idx2=$2
	local mntpt="/mnt/lustre_lfsck_${idx1}_${idx2}"
	local basedir="$mntpt/$tdir/$idx1/$idx2"

	mkdir -p $mntpt || {
		error_noexit "(11) Fail to mkdir $mntpt"
		return 11
	}

	mount_client $mntpt || {
		error_noexit "(12) Fail to mount $mntpt"
		return 12
	}

	mkdir $basedir || {
		umount_client $mntpt
		error_noexit "(13) Fail to mkdir $basedir"
		return 13
	}

	echo "&&&&& Start create $UNIT files under $basedir at: $(date) &&&&&"
	createmany -o ${basedir}/f $UNIT || {
		umount_client $mntpt
		error_noexit "(14) Fail to gen $UNIT files under $basedir"
		return 14
	}
	echo "&&&&& End create $UNIT files under $basedir at: $(date) &&&&&"

	umount_client $mntpt
}

layout_gen_set() {
	local cnt=$1

	echo "##### Start generate test set for subdirs=$cnt at: $(date) #####"
	for ((k = 0; k < $MDSCOUNT; k++)); do
		$LFS mkdir -i ${k} $LFSCKDIR/${k} || return 10

		for ((l = 1; l <= $cnt; l++)); do
			layout_gen_one ${k} ${l} &
		done
	done

	wait
	echo "##### End generate test set for subdirs=$cnt at: $(date) #####"
}

t4_test() {
	local saved_mdscount=$MDSCOUNT
	local saved_ostcount=$OSTCOUNT

	echo "stopall"
	stopall > /dev/null || error "(1) Fail to stopall"

	LFSCKDIR="$DIR/$tdir"
	MDSCOUNT=1
	for ((i = 1; i <= $saved_ostcount; i = $((i * 2)))); do
		OSTCOUNT=${i}

		echo "+++++ Start cycle ostcount=$OSTCOUNT at: $(date) +++++"
		echo

		for ((j = $MINSUBDIR; j <= $MAXSUBDIR; j = $((j * FACTOR)))); do
			echo "formatall"
			formatall > /dev/null ||
				error "(2) Fail to formatall, subdirs=${j}"

			echo "setupall"
			setupall > /dev/null ||
				error "(3) Fail to setupall, subdirs=${j}"

			mkdir $LFSCKDIR ||
				error "(4) mkdir $LFSCKDIR, subdirs=${j}"

			$LFS setstripe -c ${OSTCOUNT} -i 0 $LFSCKDIR ||
				error "(5) setstripe on $LFSCKDIR, subdirs=${j}"

			local RC=0
			layout_gen_set ${j} || RC=$?
			[ $RC -eq 0 ] ||
				error "(6) generate set $RC, subdirs=${j}"

			RC=0
			layout_test_one || RC=$?
			[ $RC -eq 0 ] ||
				error "(7) LFSCK failed with $RC, subdirs=${j}"
		done

		echo "stopall"
		stopall > /dev/null || error "(8) Fail to stopall, subdirs=${j}"

		echo
		echo "----- Stop cycle ostcount=$OSTCOUNT at: $(date) -----"
	done

	MDSCOUNT=$saved_mdscount
	OSTCOUNT=$saved_ostcount
}

test_4a() {
	t4_test
}
run_test 4a "Single MDS lfsck layout performance (routine case) without load"

test_4b() {
	echo "Inject failure stub to simulate dangling reference"
	#define OBD_FAIL_LFSCK_DANGLING 0x1610
	do_nodes $(comma_list $(osts_nodes)) $LCTL set_param fail_loc=0x1610

	t4_test
}
run_test 4b "Single MDS lfsck layout performance (repairing case) without load"

t5_test() {
	local saved_mdscount=$MDSCOUNT

	echo "stopall"
	stopall > /dev/null || error "(1) Fail to stopall"

	LFSCKDIR="$DIR/$tdir"
	for ((i = 1; i <= $saved_mdscount; i++)); do
		MDSCOUNT=${i}

		echo "+++++ Start cycle mdscount=$MDSCOUNT at: $(date) +++++"
		echo

		for ((j = $MINSUBDIR; j <= $MAXSUBDIR; j = $((j * FACTOR)))); do
			echo "formatall"
			formatall > /dev/null ||
				error "(2) Fail to formatall, subdirs=${j}"

			echo "setupall"
			setupall > /dev/null ||
				error "(3) Fail to setupall, subdirs=${j}"

			mkdir $LFSCKDIR ||
				error "(4) mkdir $LFSCKDIR, subdirs=${j}"

			$LFS setstripe -c ${OSTCOUNT} -i 0 $LFSCKDIR ||
				error "(5) setstripe on $LFSCKDIR, subdirs=${j}"

			local RC=0
			layout_gen_set ${j} || RC=$?
			[ $RC -eq 0 ] ||
				error "(6) generate set $RC, subdirs=${j}"

			RC=0
			layout_test_one || RC=$?
			[ $RC -eq 0 ] ||
				error "(7) LFSCK failed with $RC, subdirs=${j}"
		done

		echo "stopall"
		stopall > /dev/null || error "(8) Fail to stopall"

		echo
		echo "----- Stop cycle mdscount=$MDSCOUNT at: $(date) -----"
	done

	MDSCOUNT=$saved_mdscount
}

test_5a() {
	t5_test
}
run_test 5a "lfsck layout performance (routine case) without load for DNE"

test_5b() {
	echo "Inject failure stub to simulate dangling reference"
	#define OBD_FAIL_LFSCK_DANGLING 0x1610
	do_nodes $(comma_list $(osts_nodes)) $LCTL set_param fail_loc=0x1610

	t5_test
}
run_test 5b "lfsck layout performance (repairing case) without load for DNE"

lfsck_fast_create() {
	local total=$1
	local lbase=$2
	local threads=$3
	local ldir="/test-${lbase}"
	local cycle=0
	local count=$UNIT

	while true; do
		[ $count -eq 0 -o  $count -gt ${total} ] && count=$total
		local usize=$((count / NTHREADS))
		[ ${usize} -eq 0 ] && break
		local tdir=${ldir}-${cycle}-

		echo "[cycle: $cycle] [threads: $threads]"\
		     "[files: $count] [basedir: $tdir]"

		lfsck_create

		total=$((total - usize * NTHREADS))
		[ $total -eq 0 ] && break
		lbase=$((lbase + usize))
		cycle=$((cycle + 1))
	done
}

lfsck_detach_error() {
	lfsck_detach
	error "$@"
}

test_6() {
	[ $INCFACTOR -gt 25 ] && INCFACTOR=25

	echo "stopall"
	stopall > /dev/null || error "(1) Fail to stopall"

	local saved_mdscount=$MDSCOUNT

	LFSCKDIR="$DIR/$tdir"
	MDSCOUNT=1
	echo "formatall"
	formatall > /dev/null || error "(2) Fail to formatall"

	echo "setupall"
	setupall > /dev/null || error "(3) Fail to setupall"

	mkdir $LFSCKDIR || error "(4) Fail to mkdir $LFSCKDIR"

	$LFS setstripe -c ${OSTCOUNT} -i 0 $LFSCKDIR ||
		error "(5) Fail to setstripe on $LFSCKDIR"

	local RC=0
	layout_gen_set $TOTSUBDIR || RC=$?
	[ $RC -eq 0 ] ||
		error "(6) Fail to generate set $RC, subdirs=$TOTSUBDIR"

	echo
	echo "***** Start layout LFSCK on single MDS at: $(date) *****"
	$RLCTL lfsck_start -M ${MDT_DEV} -t layout -r ||
		error "(7) Fail to start layout LFSCK"

	wait_update_facet $SINGLEMDS "$LCTL get_param -n \
		mdd.${MDT_DEV}.lfsck_layout |
		awk '/^status/ { print \\\$2 }'" "completed" $WTIME || {
		show_layout 1
		error "(8) layout LFSCK cannot finished in time"
	}
	echo "***** End layout LFSCK on single MDS at: $(date) *****"

	local SPEED=$(show_layout 1 |
		      awk '/^average_speed_phase1/ { print $2 }')
	echo "lfsck_layout full_speed is $SPEED objs/sec"

	local inc_count=$((BASE_COUNT * INCFACTOR / 100))
	local nfiles=$((inc_count / 2))

	lfsck_attach
	for ((m = 0, n = $INCFACTOR; n < 100;
	      m = $((m + inc_count)), n = $((n + INCFACTOR)))); do
		local sl=$((SPEED * n / 100))

		$STOP_LFSCK > /dev/null 2>&1
		echo
		echo "start lfsck_layout with speed ${sl} at: $(date)"
		$RLCTL lfsck_start -M ${MDT_DEV} -t layout -r -s ${sl} ||
			lfsck_detach_error \
			"(9) Fail to start lfsck_layout with speed ${sl}"

		echo "&&&&& Start create files set from ${m} at: $(date) &&&&&"
		lfsck_fast_create $nfiles ${m} $NTHREADS ||
			lfsck_detach_error "(10) Fail to create files"
		echo "&&&&& End create files set from ${m} at: $(date) &&&&&"
	done

	$STOP_LFSCK > /dev/null 2>&1
	echo
	echo "start lfsck_layout with full speed at: $(date)"
	$RLCTL lfsck_start -M ${MDT_DEV} -t layout -r -s 0 ||
		lfsck_detach_error \
		"(11) Fail to start lfsck_layout with full speed"

	echo "&&&&& start to create files set from ${m} at: $(date) &&&&&"
	lfsck_fast_create $nfiles ${m} $NTHREADS ||
		lfsck_detach_error "(12) Fail to create files"
	echo "&&&&& end to create files set from ${m} at: $(date) &&&&&"

	m=$((m + inc_count))
	$STOP_LFSCK > /dev/null 2>&1
	echo
	echo "create without lfsck_layout run back-ground"
	echo "&&&&& start to create files set from ${m} at: $(date) &&&&&"
	lfsck_fast_create $nfiles ${m} $NTHREADS ||
		lfsck_detach_error "(13) Fail to create files"
	echo "&&&&& end to create files set from ${m} at: $(date) &&&&&"

	lfsck_detach
	echo
	echo "stopall"
	stopall > /dev/null || error "(14) Fail to stopall"

	MDSCOUNT=$saved_mdscount
}
run_test 6 "lfsck layout impact on create performance"

show_namespace() {
	local idx=$1

	do_facet mds${idx} \
		"$LCTL get_param -n mdd.$(facet_svc mds${idx}).lfsck_namespace"
}

namespace_test_one() {
	echo "***** Start namespace LFSCK on all devices at: $(date) *****"
	$RLCTL lfsck_start -M ${MDT_DEV} -t namespace -A -r || return 21

	for n in $(seq $MDSCOUNT); do
		wait_update_facet mds${n} "$LCTL get_param -n \
			mdd.$(facet_svc mds${n}).lfsck_namespace |
			awk '/^status/ { print \\\$2 }'" "completed" $WTIME || {
			show_namespace ${n}
			return 22
		}
	done
	echo "***** End namespace LFSCK on all devices at: $(date) *****"

	for n in $(seq $MDSCOUNT); do
		show_namespace ${n}

		local SPEED=$(show_namespace ${n} |
			      awk '/^average_speed_total/ { print $2 }')
		echo
		echo "lfsck_namespace speed on MDS_${n} is $SPEED objs/sec"
		echo
	done
}

namespace_gen_one() {
	local idx1=$1
	local idx2=$2
	local idx3=$(((idx1 + 1) % MDSCOUNT))
	local base_mntpt="/mnt/lustre_lfsck_${idx1}"
	local show_dir="$LFSCKDIR/${idx1}/${idx2}"
	local work_dir="${base_mntpt}_0/$tdir/${idx1}/${idx2}"

	mkdir $show_dir || return 20

	local count=$((UNIT * 78 / 100)) # 78% regular files
	local sub_count=$((count / NTHREADS))
	echo "Creating $count regular files under $show_dir at: $(date)"
	for ((m = 0; m < $NTHREADS; m++)); do
		local sub_dir="${base_mntpt}_${m}/$tdir/${idx1}/${idx2}"

		createmany -o ${sub_dir}/f_${m}_ $sub_count > /dev/null &
	done

	wait || {
		error_noexit "(21) Fail to gen regular files under $show_dir"
		return 21
	}

	count=$((UNIT * 3 / 100)) # 3% local sub-dirs
	echo "Creating $count local sub-dirs under $show_dir at: $(date)"
	createmany -d $work_dir/d_l_ $count > /dev/null || {
		error_noexit "(22) Fail to gen local sub-dir under $show_dir"
		return 22
	}

	# 3% * 5 = 15% regular files under local sub-dirs
	echo "Creating 5 regular files under each local sub-dir at: $(date)"
	for ((m = 0; m < $count; m++)); do
		createmany -o $work_dir/d_l_${m}/f_l_ 5 > /dev/null || {
			error_noexit \
			"(23) Fail to gen regular under $work_dir/d_l_${m}"
			return 23
		}
	done

	count=$((UNIT * 4 / 1000)) # 0.4% multiple hard-links
	echo "Creating $count multiple hard-links under $show_dir at: $(date)"
	for ((m = 0; m < $count; m++)); do
		ln $work_dir/f_0_${m} $work_dir/f_m_${m} || {
			error_noexit \
			"(24) Fail to hardlink to $work_dir/f_0_${m}"
			return 24
		}
	done

	count=$((UNIT * 3 / 1000)) # 0.3% remote sub-dirs
	echo "Creating $count remote sub-dirs under $show_dir, and 4 regular" \
		"files under each remote sub-dir at: $(date)"
	for ((m = 0; m < $count; m++)); do
		$LFS mkdir -i ${idx3} $work_dir/d_r_${m} || {
			error_noexit \
			"(25) Fail to remote mkdir $work_dir/d_r_${m}"
			return 25
		}

		# 0.3% * 4 = 1.2% regular files under remote sub-dirs
		createmany -o $work_dir/d_r_${m}/f_r_ 4 > /dev/null || {
			error_noexit \
			"(26) Fail to gen regular under $work_dir/d_r_${m}"
			return 26
		}
	done

	# 0.3% 2-striped sub-dirs + 0.6% shards of the 2-striped sub-dirs
	count=$((UNIT * 3 / 1000))
	echo "Creating $count 2-striped sub-dirs under $show_dir," \
		"and 4 regular files under each striped sub-dir at: $(date)"
	for ((m = 0; m < $count; m++)); do
		$LFS setdirstripe -i ${idx1} -c 2 -H all_char \
			$work_dir/d_s_${m} || {
			error_noexit \
			"(27) Fail to make striped-dir $work_dir/d_s_${m}"
			return 27
		}

		# 0.3% * 4 = 1.2% regular files under striped sub-dirs
		createmany -o $work_dir/d_s_${m}/f_s_ 4 > /dev/null || {
			error_noexit \
			"(28) Fail to gen regular under $work_dir/d_s_${m}"
			return 28
		}
	done
}

namespace_gen_mdt() {
	local mdt_idx=$1
	local dir_cnt=$2
	local base_mntpt="/mnt/lustre_lfsck_${mdt_idx}"

	$LFS mkdir -i ${mdt_idx} $LFSCKDIR/${mdt_idx} || return 10

	for ((m = 0; m < $NTHREADS; m++)); do
		local mntpt="${base_mntpt}_${m}"

		mkdir -p $mntpt || {
			umount ${base_mntpt}_*
			error_noexit "(11) Fail to mkdir $mntpt"
			return 11
		}

		mount_client $mntpt || {
			umount ${base_mntpt}_*
			error_noexit "(12) Fail to mount $mntpt"
			return 12
		}
	done

	for ((l = 0; l < $dir_cnt; l++)); do
		namespace_gen_one ${mdt_idx} ${l}
	done

	umount ${base_mntpt}_*
}

namespace_gen_set() {

	local cnt=$1

	echo "##### Start generate test set for subdirs=$cnt at: $(date) #####"
	for ((k = 0; k < $MDSCOUNT; k++)); do
		namespace_gen_mdt ${k} ${cnt} &
	done
	wait
	echo "##### End generate test set for subdirs=$cnt at: $(date) #####"
}

t7_test() {
	local local_loc=$1
	local saved_mdscount=$MDSCOUNT

	[ $MDSCOUNT -le 8 ] ||
		error "Too much MDT, test data set on each MDT may be unbalance"

	echo "stopall"
	stopall > /dev/null || error "(1) Fail to stopall"

	LFSCKDIR="$DIR/$tdir"
	for ((i = 2; i <= $saved_mdscount; i = $((i + 2)))); do
		MDSCOUNT=${i}

		echo "+++++ Start cycle mdscount=$MDSCOUNT at: $(date) +++++"
		echo

		for ((j = $MINSUBDIR; j <= $MAXSUBDIR;
		      j = $((j + MINSUBDIR)))); do
			echo "formatall"
			formatall > /dev/null ||
				error "(2) Fail to formatall, subdirs=${j}"

			echo "setupall"
			setupall > /dev/null ||
				error "(3) Fail to setupall, subdirs=${j}"

			mkdir $LFSCKDIR ||
				error "(4) mkdir $LFSCKDIR, subdirs=${j}"

			$LFS setstripe -c 1 -i -1 $LFSCKDIR ||
				error "(5) Fail to setstripe on $LFSCKDIR"

			do_nodes $(comma_list $(mdts_nodes)) \
				$LCTL set_param fail_loc=$local_loc

			local RC=0
			namespace_gen_set ${j} || RC=$?
			[ $RC -eq 0 ] ||
				error "(6) generate set $RC, subdirs=${j}"

			RC=0
			namespace_test_one || RC=$?
			[ $RC -eq 0 ] ||
				error "(7) LFSCK failed with $RC, subdirs=${j}"

			do_nodes $(comma_list $(mdts_nodes)) \
				$LCTL set_param fail_loc=0
		done

		echo "stopall"
		stopall > /dev/null || error "(8) Fail to stopall"

		echo
		echo "----- Stop cycle mdscount=$MDSCOUNT at: $(date) -----"
	done

	MDSCOUNT=$saved_mdscount
}

test_7a() {
	t7_test 0
}
run_test 7a "namespace LFSCK performance (routine check) without load for DNE"

test_7b() {
	echo "Inject failure stub to simulate the case of lost linkEA"
	#define OBD_FAIL_LFSCK_NO_LINKEA	0x161d
	t7_test 0x161d
}
run_test 7b "namespace LFSCK performance (repairing lost linkEA) for DNE"

test_7c() {
	echo "Inject failure stub to simulate the case of bad FID-in-dirent"
	#define OBD_FAIL_FID_INDIR      0x1501
	t7_test 0x1501
}
run_test 7c "namespace LFSCK performance (repairing bad FID-in-dirent) for DNE"

test_8() {
	[ $MDSCOUNT -lt 2 ] && skip_env "needs >= 2 MDTs"

	[ $INCFACTOR -gt 25 ] && INCFACTOR=25

	echo "stopall"
	stopall > /dev/null || error "(1) Fail to stopall"

	local saved_mdscount=$MDSCOUNT

	LFSCKDIR="$DIR/$tdir"
	MDSCOUNT=2
	echo "formatall"
	formatall > /dev/null || error "(2) Fail to formatall"

	echo "setupall"
	setupall > /dev/null || error "(3) Fail to setupall"

	mkdir $LFSCKDIR || error "(4) Fail to mkdir $LFSCKDIR"

	$LFS setstripe -c 1 -i 0 $LFSCKDIR ||
		error "(5) Fail to setstripe on $LFSCKDIR"

	local RC=0
	namespace_gen_set $TOTSUBDIR || RC=$?
	[ $RC -eq 0 ] ||
		error "(6) Fail to generate set $RC, subdirs=$TOTSUBDIR"

	echo
	echo "***** Start namespace LFSCK at: $(date) *****"
	$RLCTL lfsck_start -M ${MDT_DEV} -t namespace -A -r ||
		error "(7) Fail to start namespace LFSCK"

	for n in $(seq $MDSCOUNT); do
		wait_update_facet mds${n} "$LCTL get_param -n \
			mdd.$(facet_svc mds${n}).lfsck_namespace |
			awk '/^status/ { print \\\$2 }'" "completed" $WTIME || {
			show_namespace ${n}
			error "(8) namespace LFSCK cannot finished in time"
		}
	done
	echo "***** End namespace LFSCK at: $(date) *****"

	local SPEED=$(show_namespace 1 |
		      awk '/^average_speed_phase1/ { print $2 }')
	echo "lfsck_namespace full_speed is $SPEED objs/sec"
	echo

	local inc_count=$((BASE_COUNT * INCFACTOR / 100))
	local nfiles=$((inc_count / 2))
	local m=0

	lfsck_attach

	local stime=$(date +%s)
	lfsck_fast_create $nfiles ${m} $NTHREADS ||
		lfsck_detach_error "(9) Fail to create files"
	local etime=$(date +%s)
	echo "created $nfiles without lfsck_namespace run back-ground used" \
		"$((etime - stime)) seconds"
	echo

	for ((m = nfiles, n = $INCFACTOR; n < 100;
	      m = $((m + inc_count)), n = $((n + INCFACTOR)))); do
		local sl=$((SPEED * n / 100))

		$STOP_LFSCK > /dev/null 2>&1
		echo "start lfsck_namespace with speed ${sl} at: $(date)"
		$RLCTL lfsck_start -M ${MDT_DEV} -t namespace -A -r -s ${sl} ||
			lfsck_detach_error \
			"(10) Fail to start lfsck_namespace with speed ${sl}"

		stime=$(date +%s)
		lfsck_fast_create $nfiles ${m} $NTHREADS ||
			lfsck_detach_error "(11) Fail to create files"
		etime=$(date +%s)
		echo "created $nfiles with namespace LFSCK run with the" \
			"speed limit of ${n}% of full speed used" \
			"$((etime - stime)) seconds"
		echo
	done

	$STOP_LFSCK > /dev/null 2>&1
	echo "start lfsck_namespace with full speed at: $(date)"
	$RLCTL lfsck_start -M ${MDT_DEV} -t namespace -A -r -s 0 ||
		lfsck_detach_error \
		"(12) Fail to start lfsck_namespace with full speed"

	stime=$(date +%s)
	lfsck_fast_create $nfiles ${m} $NTHREADS ||
		lfsck_detach_error "(13) Fail to create files"
	etime=$(date +%s)
	echo "created $nfiles with namespace LFSCK run with full speed used" \
		"$((etime - stime)) seconds"
	echo

	$STOP_LFSCK > /dev/null 2>&1

	lfsck_detach

	echo "stopall"
	stopall > /dev/null || error "(14) Fail to stopall"

	MDSCOUNT=$saved_mdscount
}
run_test 8 "lfsck namespace impact on create performance"

# cleanup the system at last
lfsck_cleanup
complete $SECONDS
check_and_cleanup_lustre
exit_status


#!/bin/bash

LUSTRE=${LUSTRE:-$(dirname $0)/..}
. $LUSTRE/tests/test-framework.sh
init_test_env $@

[ -n "$LOAD" ] && load_modules && exit 0
[ -z "$NOFORMAT" ] && formatall
[ -z "$NOSETUP" ] && setupall


#!/bin/bash

LUSTRE=${LUSTRE:-$(dirname $0)/..}
. $LUSTRE/tests/test-framework.sh
init_test_env $@

cleanupall -f


#!/bin/bash

LUSTRE=${LUSTRE:-$(dirname $0)/..}
. $LUSTRE/tests/test-framework.sh
init_test_env $@
init_logging

ALWAYS_EXCEPT="$LNET_SELFTEST_EXCEPT"
if (( $LINUX_VERSION_CODE >= $(version_code 4.4.0) )); then
	# bug number for skipped test: LU-10073
	ALWAYS_EXCEPT+="               smoke "
fi

if [[ $(uname -m) = ppc64 ]]; then
	# bug number for skipped test: LU-10073
	ALWAYS_EXCEPT+="               smoke "
fi

build_test_filter

[ x$LST = x ] && skip_env "lst not found LST=$LST"

# FIXME: what is the reasonable value here?
lst_LOOP=${lst_LOOP:-100000}
lst_CONCR=${lst_CONCR:-"1 2 4 8"}
lst_SIZES=${lst_SIZES:-"4k 8k 256k 1M"}
if [ "$SLOW" = no ]; then
    lst_CONCR="1 8"
    lst_SIZES="4k 1M"
    lst_LOOP=1000
fi

smoke_DURATION=${smoke_DURATION:-1800}
if [ "$SLOW" = no ]; then
    [ $smoke_DURATION -le 300 ] || smoke_DURATION=300
fi

lst_TESTS=${lst_TESTS:-"write read ping"}

# "none" -> LST_BRW_CHECK_NONE
# "full" -> LST_BRW_CHECK_FULL
# "simple" -> LST_BRW_CHECK_SIMPLE
lst_CHECK=${lst_CHECK:-"full"}

lst_FROM=${lst_FROM:-"cs"}

case $lst_CHECK in
	full|simple) check="check=$lst_CHECK";;
	none) check="";;
	*) error Unknown flag $lst_CHECK;;
esac
nodes=$(comma_list "$(osts_nodes) $(mdts_nodes)")
lst_SERVERS=${lst_SERVERS:-$(comma_list "$(host_nids_address $nodes $NETTYPE)")}
lst_CLIENTS=${lst_CLIENTS:-$(comma_list "$(host_nids_address $CLIENTS $NETTYPE)")}
interim_umount=false
interim_umount1=false

#
# _restore_mount(): This function calls restore_mount function for "MOUNT" and
# "MOUNT2" paths to mount clients if they were not mounted and were umounted
# in this file earlier.
# Parameter: None
# Returns: None. Exit with error if client mount fails.
#
_restore_mount () {
	if $interim_umount && ! is_mounted $MOUNT; then
		restore_mount $MOUNT || error "Restore $MOUNT failed"
	fi

	if $interim_umount1 && ! is_mounted $MOUNT2; then
		restore_mount $MOUNT2 || error "Restore $MOUNT2 failed"
	fi
}

if local_mode; then
   lst_SERVERS=`hostname`
   lst_CLIENTS=`hostname`
fi

# FIXME: do we really need to unload lustre modules on all nodes?
# bug 19387, comment 9
# unloading lustre modules is not strictly necessary but unmounting
# /mnt/lustre before running lst would be useful:
# 1) because lustre messages clutter logs - we needn't them for testing LNET
# 2) it's theoretically possible that lst tests congest comm paths so tightly
# that mounted lustre wouldn't able to perform some of its background activities
if is_mounted $MOUNT; then
	cleanup_mount $MOUNT || error "Fail to unmount client $MOUNT"
	interim_umount=true
fi

if is_mounted $MOUNT2; then
	cleanup_mount $MOUNT2 || error "Fail to unmount client $MOUNT2"
	interim_umount1=true
fi

lst_prepare () {
    # Workaround for bug 15619
    lst_cleanup_all
    lst_setup_all
}

# make batch
test_smoke_sub () {
    local servers=$1
    local clients=$2

    local nc=$(echo ${clients//,/ } | wc -w)
    local ns=$(echo ${servers//,/ } | wc -w)
    echo '#!/bin/bash'
    echo 'set -e'

    echo 'cleanup () { trap 0; echo killing $1 ... ; kill -9 $1 || true; }'

    echo "$LST new_session --timeo 100000 hh"
    echo "$LST add_group c $(nids_list $clients)"
    echo "$LST add_group s $(nids_list $servers)"
    echo "$LST add_batch b"

	declare -a tests

	case $lst_FROM in
		c) tests[0]="${nc}:${ns} --from c --to s";;
		s) tests[0]="${ns}:${nc} --from s --to c";;
		cs)tests[0]="${nc}:${ns} --from c --to s"
		   tests[1]="${ns}:${nc} --from s --to c";;
		*) error Unknown flag $lst_FROM;;
	esac

	pre="$LST add_test --batch b --loop $lst_LOOP "
	for t in $lst_TESTS; do
		for s in $lst_SIZES; do
			for c in $lst_CONCR; do
				for ((i=0; i<${#tests[@]}; i++)); do
					echo -n "$pre --concurrency $c"\
						" --distribute ${tests[i]} "
					case $t in
						read|write)
							echo -n "brw $t" \
							" $check size=$s";;
						ping)
							echo -n $t;;
						*) error Unknonwn LST test;;
					esac
					echo
				done
			done
		done
	done

    echo $LST run b
    echo sleep 1
    echo "$LST stat --delay 10 --timeout 10 c s &"
    echo 'pid=$!'
    echo 'trap "cleanup $pid" INT TERM'
    echo sleep $smoke_DURATION
    echo 'cleanup $pid'

}

run_lst () {
   local file=$1

   export LST_SESSION=$$

   # start lst
   sh $file
}

check_lst_err () {
	local log=$1

	grep ^Total $log

	if awk '/^Total.*nodes/ {print $2}' $log | grep -vq '^0$'; then
		_restore_mount
		error 'lst Error found'
	fi
}

test_smoke () {
	lst_prepare

	local servers=$lst_SERVERS
	local clients=$lst_CLIENTS

	local runlst=$TMP/smoke.sh

	local log=$TMP/$tfile.log
	local rc=0

	test_smoke_sub $servers $clients 2>&1 > $runlst

	cat $runlst

	run_lst $runlst | tee $log
	rc=${PIPESTATUS[0]}
	[ $rc = 0 ] || { _restore_mount; error "$runlst failed: $rc"; }

	lst_end_session --verbose | tee -a $log

	# error counters in "lst show_error" should be checked
	check_lst_err $log
	lst_cleanup_all
}
run_test smoke "lst regression test"

complete $SECONDS
_restore_mount
check_and_cleanup_lustre
exit_status


#!/bin/sh
set -e

export PATH=`dirname $0`:`dirname $0`/../utils:$PATH
CREATEMANY=${CREATEMANY:-createmany}
STATMANY=${STATMANY:-statmany}
UNLINKMANY=${UNLINKMANY:-unlinkmany}
LCTL=${LCTL:-lctl}

MOUNT=${MOUNT:-/mnt/lustre}
MOUNT2=${MOUNT2:-/mnt/lustre2}
DIR=${DIR:-$MOUNT}
DIR2=${DIR2:-$MOUNT2}
COUNT=${COUNT:-100}

cleanup() {
	[ $CR_PID ] && kill -9 $CR_PID
	[ $ST_PID ] && kill -9 $ST_PID
}

trap cleanup EXIT

LOCKDIR=$DIR/lockdir
LOCKFILE=$LOCKDIR/lockfile
rm -rf $LOCKDIR

NUM=0

MINDIR=$DIR
MAXDIR=$DIR
MINRES=4294967295
MAXRES=0
mkdir -p $MINDIR
while [ $MINRES -gt $MAXRES ]; do
	FILETMP=$MINDIR/f$$${NUM}
	DIRTMP=$DIR/d$$/d${NUM}
	touch $FILETMP
	mkdir -p $DIRTMP
	FILERES=`ls -id $FILETMP | awk '{ print $1 }'`
	DIRRES=`ls -id $DIRTMP | awk '{ print $1 }'`
	if [ $DIRRES -gt $MAXRES ]; then
		MAXDIR=$DIRTMP
		MAXRES=$DIRRES
	fi
	if [ $FILERES -lt $MINRES -o -z "$MINFILE" ]; then
		[ -f "$MINFILE" ] && rm $MINFILE
		MINFILE=$FILETMP
		MINRES=$FILERES
	else
		rm $FILETMP
	fi
	NUM=$(($NUM + 1))
done

mv $MAXDIR $LOCKDIR
mv $MINFILE $LOCKFILE
rm -rf $DIR/d$$

$LCTL mark "start dir: $LOCKDIR=$MAXRES file: $LOCKFILE=$MINRES"
# link will lock $LOCKFILE and $DIR as it creates ${LOCKFILE}{0,1,...}
$CREATEMANY -l$LOCKFILE $LOCKFILE -$COUNT &
CR_PID=$!

while ! test -f ${LOCKFILE}1 ; do
	sleep 1
done

# this will lock $DIR and ${LOCKFILE}0
$STATMANY -s $DIR2/lockdir/lockfile 1 -$COUNT &
ST_PID=$!

sleep $(($COUNT / 2))

$UNLINKMANY $DIR2/lockdir/lockfile 1 $(($COUNT * 1000)) || true

trap 0
kill $CR_PID || true
kill $ST_PID || true

rm -rf $LOCKDIR