#!/sbin/sh -
# %W% %G% %U% - %Q%
#ident "%Z%unixvm:%M% %I%"

# Copyright (c) 2000 VERITAS Software Corporation.  ALL RIGHTS RESERVED.
# UNPUBLISHED -- RIGHTS RESERVED UNDER THE COPYRIGHT
# LAWS OF THE UNITED STATES.  USE OF A COPYRIGHT NOTICE
# IS PRECAUTIONARY ONLY AND DOES NOT IMPLY PUBLICATION
# OR DISCLOSURE.
# 
# THIS SOFTWARE CONTAINS CONFIDENTIAL INFORMATION AND
# TRADE SECRETS OF VERITAS SOFTWARE.  USE, DISCLOSURE,
# OR REPRODUCTION IS PROHIBITED WITHOUT THE PRIOR
# EXPRESS WRITTEN PERMISSION OF VERITAS SOFTWARE.
# 
#               RESTRICTED RIGHTS LEGEND
# USE, DUPLICATION, OR DISCLOSURE BY THE GOVERNMENT IS
# SUBJECT TO RESTRICTIONS AS SET FORTH IN SUBPARAGRAPH
# (C) (1) (ii) OF THE RIGHTS IN TECHNICAL DATA AND
# COMPUTER SOFTWARE CLAUSE AT DFARS 252.227-7013.
#               VERITAS SOFTWARE
# 1600 PLYMOUTH STREET, MOUNTAIN VIEW, CA 94043


__VXVM_CONNECT_WAIT=WAIT
export __VXVM_CONNECT_WAIT

#
# Need to force C locale setting so that
# english text is processed.
#
ENV_LANG=$LANG
ENV_LC_MESSAGES=$LC_MESSAGES
LANG=C
export LANG
LC_MESSAGES=C
export LC_MESSAGES


: ${VOLROOT_DIR:=$__VXVM_ROOT_DIR}
. ${VOL_SCRIPTS_LIB:-$VOLROOT_DIR/usr/lib/vxvm/lib}/vxcommon

prog=vxrelocd

if [ -f /usr/SUNWale/bin/mailx ]
then
	MAILER=/usr/SUNWale/bin/mailx
else
	MAILER=mailx
fi



set_to_env_locale()
{
	LANG=$ENV_LANG
	export LANG
	LC_MESSAGES=$ENV_LC_MESSAGES
	export LC_MESSAGES
}

set_to_c_locale()
{
	LANG=C
	export LANG
	LC_MESSAGES=C
	export LC_MESSAGES
}

usage()
{
	set_to_env_locale

	export prog; egettxt >&2 \
"Usage: $prog [-o vxrecover_arg] [-O old_version] [-s save_max] [mail-address ...]" \
	vxvmshm:956
	exit 1
}

# Find all dead volumes that have storage on a speficic DM
deadvols()
{
	dmname=$1
	dgname=$2
	shift 2

	vxprint -g $dgname -nv -e 'v_read_pol!=V_RAID &&
			!(any aslist.pl_kstate=ENABLED) &&
			(any aslist.aslist.sd_dmname="'$dmname'")'
}

deadraids()
{
	dmname=$1
	dgname=$2
	shift 2

	vxprint -g $dgname -p -F"%vname" \
		-e '(any aslist.sd_dmname="'$dmname'") &&
		    pl_layout=RAID && assoc.v_unusable'	
}

# Find all affected volumes that have storage on a specific DM but
# are not completely bashed
affvols()
{
	dmname=$1
	dgname=$2
	shift 2

	vxprint -g $dgname -nv -e '(v_read_pol!=V_RAID) && (any aslist.pl_kstate=ENABLED) && (any aslist.aslist.sd_dmname="'$dmname'")'

	vxprint -g $dgname -p -F'%vname' \
		-e '(any aslist.sd_dmname="'$dmname'") &&
		    pl_layout=RAID && !assoc.v_unusable'
}

o_args=
check_cnt=2
oldhot=
savemax=

while getopts :vs:O:o: c
do
	case $c in
	s)	savemax=$OPTARG;;
	O)	oldhot=$OPTARG;;
	o)	o_args=$OPTARG;;
	v)	verbose="on";;
	?)	usage;;
	esac
done

case "$oldhot" in
	"1"* )		oldhot=pre3.1;;
	"2"* )		oldhot=pre3.1;;
	"3.0"* )	oldhot=pre3.1;;
esac

# maximum number of config files to be kept per disk group
[ "X$savemax" = "X" ] && savemax=32
[ X`echo $savemax | awk '/^[0-9]+$/ {print $1}'` = "X" ] && usage

shift `expr $OPTIND - 1`
[ $# -eq 0 ] && set root

recipients="$@"

shift $#

# set the da_name for root disk
set_OS_variables

# check c#t#d# format
hr_checkctd()
{
    if	[ "X`echo $1 | \
			awk '/^m?c[0-9]+t[0-9]+d[0-9]+s[0-9]+$/ {print $1}'`" != "X" ] ||
		[ "X`echo $1 | \
			awk '/^m?c[0-9]+t[0-9]+d[0-9]+$/ {print $1}'`" != "X" ] ||
		[ "X`echo $1 | \
			awk '/^m?c[0-9]+[tds][0-9]+$/ {print $1}'`" != "X" ] ||
		[ "X`echo $1 | \
			awk '/^m?c[0-9]+$/ {print $1}'`" != "X" ]
	then
		return 0
	else
		return 1
	fi
}

# display da_name in c#t#d#s# format
hr_da2ctds()
{
	# check for the ctds format
	ctdpat=$1
	hr_checkctd "$ctdpat"
    if [ $? -eq 0 ]; then
        echo $ctdpat
        return
    fi

	# get CTD info for EMC disks
	if [ -s /etc/powermt ]; then
		emcdevno=`echo $ctdpat | tr -d [a-z] | tr -d [A-Z]`
		activedas=`/etc/powermt display dev=$emcdevno 2> /dev/null | \
						awk '$3=="Active" || $3=="Standby" {print $4}'`
		if [ "X$activedas" != "X" ]; then
			for emcda in $activedas; do
				hr_checkctd "$emcda"
    			if [ $? -eq 0 ]; then
					vxdisk list $emcda > /dev/null 2>&1
					if [ $? -eq 0 ] ; then
						echo $emcda
						return
					fi      
				fi
			done
		fi
	fi

	# get controller info
	ctlr_num=c`vxdisk getctlr $ctdpat 2> /dev/null `
	if [ $? -eq 0 ] && [ "X$ctlr_num" != "Xc-1" ] ; then
		hr_checkctd "$ctlr_num"
		if [ $? -eq 0 ]; then
			echo $ctlr_num
			return
		fi
	fi

	# if got here, da_name format is different from cNtNdN format.
	# relocation still work but disk selection based on CTD not supported.
    echo "-"
}

hr_add_avail_disks()
{
	devpat=$1
	> $hr_tmp2
	> $hr_tmp3
	
	# arranging dms, putting active dms first
	cat $hr_tmp1 | egrep "^m?${devpat}([a-z]+|-|$| |	)" | \
	awk '{print $2}' | \
	while read a; do
		for b in $voldms ; do
			if [ "X$a" = "X$b" ]; then
				echo $a >> $hr_tmp2
			else
				echo $a >> $hr_tmp3
			fi
		done
	done

	cat $hr_tmp2 $hr_tmp3 | sed -e '/^$/d' | \
	while read cdm; do
		[ X`cat $hr_availdms | awk '$1=="'$cdm'" {print $1}'` != "X" ] &&
			continue
		vxdisk -g $failed_dg check $cdm > /dev/null 2>&1
		[ $? -eq 0 ] && echo $cdm >> $hr_availdms
	done
}

# get list of disks that are marked as spares/free for hot-relocation,
# sorted by the minimal distance to the failed dm

hr_list_avail_disks()
{
	spareflag=$1
	freeuseflag=$2
	hr_availdms=$3

	> $hr_tmp1
	> $hr_tmp2
	> $hr_availdms

	if [ "X`vxprint -g $failed_dg -dF %nohotuse $failed_dm`" = "X-" ]; then
		[ "X$freeuseflag" = "Xoff" ] && freeuseflag="-"
	fi

	vxprint -g $failed_dg -Qqd -F \
		'%name %daname %spare %nohotuse %nodarec %failing %reserve' | \
		awk '$3=="'$spareflag'" && $4=="'$freeuseflag'" && $5=="off" && \
			$7=="off" && \
			$6=="off" && $1!="'$failed_dm'" {print $2 " " $1}' > \
			$hr_tmp2 2> /dev/null
	[ ! -s $hr_tmp2 ] && return

	cat $hr_tmp2 | \
	while read da dm; do
		echo `hr_da2ctds $da` $dm >> $hr_tmp1
	done

	failed_da=`vxprint -g $failed_dg -Qqd -F '%last_da_name' $failed_dm `
	failed_da=`hr_da2ctds $failed_da | sed 's/mc/c/g' `
	voldms=`vxprint -g $failed_dg -Qqse 'assoc.assoc.v_name="'$failed_v'"' \
				-F %dmname 2> /dev/null`

	# get disks with same controller, target, and devs
	hr_add_avail_disks `echo $failed_da | awk '{FS="s"; print $1}'`

	# get disks with same controller, target
	hr_add_avail_disks `echo $failed_da | awk '{FS="d"; print $1}'`

	# get disks with same controller
	hr_add_avail_disks `echo $failed_da | awk '{FS="t"; print $1}'`

	# get everything else
	hr_add_avail_disks ""
}

# go through the list and find disks with enough free space
# space for root and swap should be contig

hr_select_spacedm()
{
	ifaillen=$1
	rootlen=$2
	swaplen=$3
	dmlist=$4
	spacedms=$5

	> $spacedms
	cat $dmlist | \
	while read fldm; do
		[ X`cat $spacedms | awk '$1=="'$fldm'" {print $1}'` != "X" ] &&
			continue
		freespace=`vxdg -g $failed_dg -qa free $fldm 2> /dev/null | \
						awk '{sum+=$5} END {printf "%d\n", 0+sum}'`
		[ "X$freespace" = "X" ] && continue
		if [ $freespace -ge $ifaillen ]; then
			allchunks=`vxdg -aqg $failed_dg free $fldm | \
							awk '{printf "%d\n", 0+$5}' | sort -nr `
			chunk1=`echo $allchunks | awk '{print $1}'`
			chunk2=`echo $allchunks | awk '{print $2}'`
			if	[ `expr $rootlen + $swaplen` -le $chunk1 ] || \
				[ $rootlen -le $chunk1 -a $swaplen -le $chunk2 ] || \
				[ $rootlen -le $chunk2 -a $swaplen -le $chunk1 ]; then
				echo $fldm >> $spacedms
			fi
		fi
	done
}

# find a replacement dm where failed SDs can go without any conflicts with
# the existing SDs on it.

hr_get_cleardm()
{
	hr_failvp=$1
	spacedms=$2

	cat $spacedms | \
	while read newdm ; do
		> $hr_tmp2
		# list existing volumes and plexes whose SDs are on a dm
		> $hr_newvp
		vxprint -g $failed_dg -Qqse 'sd_dmname="'$newdm'" ' \
				-F '%v_name %pl_name' 2> /dev/null | \
				sed -e '/^$/d' | sort | uniq > $hr_newvp

		# check for overlap/conflict between a failed plex and a
		# plex that is existing on a disk intended for relocation

		cat  $hr_newvp | \
		while read newdmv newdmp; do

			# skip if not associated.
			[ "X$newdmv" = "X-" ] && continue

			cat $hr_failvp | \
			while read failv failp ; do

				# if layered vols, get the top common plex and its layout.
				nlayerv=$newdmv
				nlayerp=""
				while [ "X$nlayerv" != "X" ]; do
					nlayerp="$nlayerp "`vxprint -g $failed_dg -Qse \
										'sd_dmname="'$nlayerv'"' -F %pl_name`
					nlayerv=`vxprint -g $failed_dg -Qse \
										'sd_dmname="'$nlayerv'"' -F %v_name`
				done

				faillyp=""
				newlyp=""
				faillylayout=""
				flayerv=$failv
				while [ "X$flayerv" != "X" ]; do
					flayerp=`vxprint -g $failed_dg -Qse \
										'sd_dmname="'$flayerv'"' -F %pl_name`
					for cplex in $nlayerp; do
						if [ "$cplex" = "$flayerp" ]; then
							faillylayout=`vxprint -g $failed_dg \
											-F '%layout' "$cplex" `
							newlyp=$cplex
							faillyp=$cplex
							break 2
						fi
					done
					flayerv=`vxprint -g $failed_dg -Qse \
										'sd_dmname="'$flayerv'"' -F %v_name`
				done

				# if sds are part of the same volume, check for overlap.
				if [ "$newdmv" = "$failv" ] || \
					[ "X$newlyp" = "X$faillyp" -a "X$newlyp" != "X" ]; then

					faillayout=`vxprint -g $failed_dg -F '%layout' "$failp" `

					# 0 = plex is not a DRL log, 1 = yes.
					newlog=X`vxprint -g $failed_dg -Qpe 'pl_name="'$newdmp'" &&
							aslist.sd_is_log' -F %name | \
							wc -l | awk '{print $1}' `
					faillog=X`vxprint -g $failed_dg -Qpe 'pl_name="'$failp'" &&
							aslist.sd_is_log' -F %name | \
							wc -l | awk '{print $1}' `

					# a failed sd can go to a dm if there is no conflict with
					# another sd on that dm and all are false:
					# 1. sds of the same plex, plex!=concat
					# 2. sds of a different plex of the same volume and
					#		!(one plex is a DRL log)
					# 3. sds of a different plex and volume, same non-CONCAT
					# 		super-plex and both sub-plexes are not DRL logs

	 				if  [ "$newdmp" = "$failp" -a \
							"$faillayout" != "CONCAT" ] || \
						[ "$newdmp" != "$failp" -a \
							"$newlog" = "$faillog" -a \
							"$newdmv" = "$failv" ] || \
						[ "$newdmv" != "$failv" -a \
							"$faillylayout" != "CONCAT" -a \
							"$newlog" = "X0" -a \
							"$faillog" = "X0" ]; then
						echo $newdmp > $hr_tmp2
						break
 					fi
				fi
			done
			[ -s $hr_tmp2 ] && break
		done
		if [ ! -s $hr_tmp2 ]; then
			echo "$newdm"
			break
		fi
	done
}

# saving the current config to a file before attempting to hot-relocate

hr_save_config()
{
	[ $savemax -eq 0 ] && return

	# one save per trip
	[ -f $failed_dir/$failed_dg.$failed_file ] && return

	[ -d $failed_dir ] || mkdir -p $failed_dir

	touch $failed_dir/$failed_dg.$failed_file > $hr_tmp4 2>&1

	if [ $? -ne 0 ]; then
		(export hr_tmp4 failed_dir failed_dg failed_file ; egettxt "\
\\nVERITAS Volume Manager failed to save the current configuration in:
$failed_dir/$failed_dg.$failed_file
\\n" vxvmshm:976
cat $hr_tmp4
		) | $MAILER -s "$_ms" "$recipients"
		return
	else
		(export failed_dir failed_dg failed_file ; egettxt "\
\\nVERITAS Volume Manager is preparing to relocate for diskgroup $failed_dg.
Saving the current configuration in:
$failed_dir/$failed_dg.$failed_file
\\n" vxvmshm:977
		) | $MAILER -s "$_ms" "$recipients"
	fi

	echo "#vxprint -g $failed_dg -ht" > \
			$failed_dir/$failed_dg.$failed_file 2>&1
	vxprint -g $failed_dg -ht    >> $failed_dir/$failed_dg.$failed_file 2>&1

	echo "#vxprint -g $failed_dg -mpvsh" >> \
			$failed_dir/$failed_dg.$failed_file 2>&1
	vxprint -g $failed_dg -mpvsh >> $failed_dir/$failed_dg.$failed_file 2>&1

	# remove the oldest and keep only the most recent
	savecount=`ls -l $failed_dir/$failed_dg.* 2> /dev/null | wc -l`
	if [ $savecount -gt $savemax ]; then
		rm -f `ls -lrt $failed_dir/$failed_dg.* | \
				sed -n '1p' | awk '{print $NF}' `
	fi
}

# save target dm into history file
hr_put_hist()
{
	histdm=$1
	[ "X$histdm" = "X" ] && return
	findres=`cat $hr_hist | \
			awk '$1=="'$failed_dm'" && $2=="'$histdm'" && \
				$3=="'$failed_dg'" {print $1}' `
	[ "X$findres" = "X" ] && echo $failed_dm $histdm $failed_dg >> $hr_hist
}

# unrelocate flags don't exist in diskgroup versions before 70.
hr_set_unreloc_flag()
{
	dgversion=`vxdg list $1 | grep "^version" | awk '{print $2}'`
	if [ -z "$dgversion" ] || [ "$dgversion" -lt "70" ]; then
		eval "${4}="
	else
		eval "${4}=\"${2} ${3}\""
	fi
}

# if the moved volume is a swap and is on top of a dump device,
# reconfig the device.
hr_conf_dumpdev()
{
	_ckdg=$1
	_ckvol=$2
	_fdumpdm=$3
	_ndumpdm=$4
	[ -n "$_ckdg" -a -n "$_ckvol" -a -n "$_fdumpdm" -a -n "$_ndumpdm" ] ||
		return
	# reconfig dump device only if volume is a swap
	[ "X`vxprint -g $_ckdg -F %use_type $_ckvol`" = "Xswap" ] || return

	# get current dump partition
	if [ -x /usr/sbin/dumpadm ]; then
		# nothing to do if savecore is off
		[ "X`dumpadm | grep \"Savecore enabled:\" | \
			awk '{print $3}'`" != "Xyes" ] && return
		_curr_dump_part=`dumpadm | grep "Dump device:" | awk '{print $3}'`
	else
		_curr_dump_part=`echo "dumpfile+0x10/s" |
							adb -k /dev/ksyms /dev/mem |
							grep "^dumpfile" | awk '{print $2}'`
	fi
	[ -n "$_curr_dump_part" -a "X$_curr_dump_part" != "Xnone" ] || return
	_orig_dumpdev=`subdisk_to_partition  $_ckdg $_ckvol $_fdumpdm`

	# skip if the underlying partition is different from the dump device
	if [ -n "$_orig_dumpdev" ] && [ "$_orig_dumpdev" != "$_curr_dump_part" ]
	then
		return
	fi

	# get current dump path
	dogi_path_to_slice $_curr_dump_part _curr_dump_slice
	dogi_name_is_slice $_curr_dump_slice || return
	dogi_slice_to_device $_curr_dump_slice _curr_dump_device
	dogi_device_blkpath $_curr_dump_device _curr_dump_blkpath
	[ -n "$_curr_dump_blkpath" ] || return

	# get block path of the failed disk
	_orig_dump_slice=`vxprint -g $_ckdg -F %last_da_name $_fdumpdm`
	dogi_name_is_slice $_orig_dump_slice || return
	dogi_slice_to_device $_orig_dump_slice _orig_dump_device
	dogi_device_blkpath $_orig_dump_device _orig_dump_blkpath
	[ -n "$_orig_dump_blkpath" ] || return
	[ "$_curr_dump_blkpath" = "$_orig_dump_blkpath" ] || return

	_new_dumpdev=`subdisk_to_partition  $_ckdg $_ckvol $_ndumpdm`
	[ -n "$_new_dumpdev" ] || return

	set_to_env_locale

	if [ ! -x /usr/sbin/dumpadm ]; then
	(export _ckvol _curr_dump_part _fdumpdm _ndumpdm _ckdg ; egettxt "\
\\nVolume $_ckvol of disk group $_ckdg was moved from disk $_fdumpdm
to disk $_ndumpdm. A dump device $_curr_dump_part
was found on disk $_fdumpdm. A reboot is needed to reconfigure
the device.
\\n" vxvmshm:1269
	) | $MAILER -s "$rootdm_ms" "$recipients"
		set_to_c_locale
		return
	fi

	# set dump device to new partition
	dumpadm -d $_new_dumpdev > /dev/null 2>&1

	if [ $? -eq 0 ]; then
	(export _ckvol _fdumpdm _ndumpdm _ckdg _curr_dump_part _new_dumpdev
	egettxt "\
\\nVolume $_ckvol of disk group $_ckdg was moved from disk $_fdumpdm
to disk $_ndumpdm. A dump device $_curr_dump_part
was found on disk $_fdumpdm and was reconfigured
to $_new_dumpdev.
\\n" vxvmshm:1268
	) | $MAILER -s "$rootdm_ms" "$recipients"
	else
	(export _ckvol _fdumpdm _ndumpdm _ckdg _curr_dump_part _new_dumpdev
	egettxt "\
\\nVolume $_ckvol of disk group $_ckdg was moved from disk $_fdumpdm
to disk $_ndumpdm. A dump device $_curr_dump_part
was found on disk $_fdumpdm. Attempt to reconfigure
the device to $_new_dumpdev failed.
\\n" vxvmshm:1270
	) | $MAILER -s "$rootdm_ms" "$recipients"
	fi

	set_to_c_locale
}

# If an error occurs on the rootdisk, another disk will be selected and
# all vols will be moved to the new disk. If the entired rootdisk can't be
# moved then nothing will be done.

hr_move_rootdm()
{
	rootdm_ms="VxVM relocation of rootdisk on host `uname -n`"

	# If there is a disabled vol, then nothing will be moved.
	failrootvs=""
	for a in `vxprint -g $failed_dg -ve \
				'aslist.aslist.sd_dmname="'$failed_dm'"' -F %name `; do
		goodpl_name=`vxprint -g $failed_dg -pe 'pl_state!="LOG"
					&& pl_v_name=="'$a'" && pl_kstate==ENABLED
					&& pl_state !~ /SNAP/ && !pl_log' -F %name`
		if [ "X$goodpl_name" = "X" ] ||
			[ X`vxprint -g $failed_dg -ve 'v_name="'$a'" && \
					v_kstate!=ENABLED' -F %name` = "X$a" ]; then
			failrootvs="$failrootvs $a"
		fi
	done

	set_to_env_locale

	if [ "X$failrootvs" != "X" ]; then
		(export failed_dm failed_dg failrootvs ; egettxt "\
\\nRootdisk $failed_dm of disk group $failed_dg experienced a failure,
but no relocation will be done because its volumes:
$failrootvs
have no suitable mirror from which to recover data.
\\n" vxvmshm:972
		) | $MAILER -s "$rootdm_ms" "$recipients"
		set_to_c_locale
		return 11
	fi

	set_to_c_locale

	# get list of plexes to move with root and swap vols first so to get
	# the contiguous allocs, larger plex first

	> $hr_failvp
	rootlen=`vxprint -g $failed_dg -Qpe 'aslist.sd_dmname="'$failed_dm'" &&
				assoc.v_use_type="root"' -F '%len %v_name %pl_name' | \
				awk '{print $2 " " $3 | " sort | uniq >> '$hr_failvp'";
						sum+=$1} END {printf "%d", 0+sum}' `
	swaplen=`vxprint -g $failed_dg -Qpe 'aslist.sd_dmname="'$failed_dm'" &&
				assoc.v_use_type="swap"' -F '%len %v_name %pl_name' | \
				awk '{print $2 " " $3 | " sort | uniq >> '$hr_failvp'";
						sum+=$1} END {printf "%d", 0+sum}' `
	[ "X$swaplen" = "X" ] && swaplen=0
	[ "X$rootlen" = "X" ] && rootlen=0

	if [ $swaplen -gt $rootlen ]; then
		cat $hr_failvp > $hr_tmp1
		cat $hr_tmp1 | grep -n '.*' | sort -n -r | \
				sed 's/^.*://' > $hr_failvp
	fi

	# list of log plexes to be moved.
	logslen=`vxprint -g $failed_dg -se 'sd_dmname="'$failed_dm'" &&
					(assoc.pl_state="LOG" || assoc.pl_state="BADLOG" ||
					assoc.pl_state="SNAPDONE" || assoc.pl_state="SNAPATT" ||
					sd_is_log) ' -F '%len %v_name %pl_name' | \
					awk '{print $2 " " $3 | " sort | uniq >> '$hr_failvp'"
							sum+=$1} END {printf "%d", 0+sum}' `
	[ "X$logslen" = "X" ] && logslen=0

	# list all other plexes
	movelen=`vxprint -g $failed_dg -se 'sd_dmname="'$failed_dm'" &&
					assoc.pl_state!="LOG" && assoc.pl_state!="BADLOG" &&
					assoc.pl_state!="SNAPDONE" && assoc.pl_state!="SNAPATT" &&
					!sd_is_log && assoc.assoc.v_use_type!="root" &&
					assoc.assoc.v_use_type!="swap" ' \
					-F '%len %v_name %pl_name' | \
					awk '{print $2 " " $3 | " sort | uniq >> '$hr_failvp'"
							sum+=$1} END {printf "%d", 0+sum}' `
	[ "X$movelen" = "X" ] && movelen=0
	movelen=`expr $movelen + $rootlen + $swaplen + $logslen`
	[ $movelen -eq 0 ] && return 0

	# get list of spares and disks with nohotuse flags not set
	hr_list_avail_disks on  off $hr_sparedms

	# check if spare=only flag is set from the attribute defaults
	if [ -s $VOL_DEFAULTS_FILE ]; then
		dsparefl=`cat $VOL_DEFAULTS_FILE | awk '{FS="#"; print $1}' | \
						awk ' {for (i=1; i<=NF; i++)
							if ($i ~ /^spare=/) { svar=$i }
							} END {print svar} ' `
	fi

	# non-spares can be used if not marked with nohotuse flag
	if [ "X$dsparefl" != "Xspare=only" ]; then
		hr_list_avail_disks off off $hr_nofuodms
		cat $hr_nofuodms >> $hr_sparedms
	fi

	# select disks with enough space
	hr_select_spacedm $movelen $rootlen $swaplen $hr_sparedms $hr_fitdms

	# select disk with no conflicts
	cleardm=`hr_get_cleardm "$hr_failvp" "$hr_fitdms" `

	set_to_env_locale

	if [ "X$cleardm" = "X" ]; then
		(export failed_dm failed_dg; egettxt "\
\\nRootdisk $failed_dm of disk group $failed_dg experienced a failure,
but there is no suitable disk found to relocate
all associated subdisks.
\\n" vxvmshm:973
		) | $MAILER -s "$rootdm_ms" "$recipients"
		set_to_c_locale
		return 12
	else
	(export failed_dm failed_dg cleardm ; egettxt "\
\\nRootdisk $failed_dm of disk group $failed_dg experienced a failure,
hot-relocation is attempting to move all associated subdisks to $cleardm
\\n" vxvmshm:975
	) | $MAILER -s "$rootdm_ms" "$recipients"
	fi

	# save current config
	hr_save_config

	> $hr_rtdone
	> $hr_rtfail

	set_to_c_locale

	cat $hr_failvp | \
	while read movev movep ; do
		# skip if not associated or moved already.
		[ "X$movev" = "X-" ] && continue
		if [ X`vxprint -g $failed_dg -pe 'aslist.sd_dmname="'$failed_dm'" &&
				pl_name="'$movep'" ' -F %name ` = "X" ]; then
			continue
		fi

		relc_ret=0
		plusetype=`vxprint -g $failed_dg -p -F %use_type $movep`

		# remirror root and swap
		if [ "X$plusetype" = "Xroot" -o "X$plusetype" = "Xswap" ]; then

			hr_set_unreloc_flag "$failed_dg" "-l" "$movep" hr_unreloc_flag

			vxassist -g $failed_dg -o nofmr $hr_unreloc_flag mirror \
			      $movev layout=nospan,contig $cleardm 2> /dev/null

			move_ret=$?
			if [ $move_ret -eq 0 ] ; then
				/etc/vx/bin/vxbootsetup
				# reconfig dump device
				[ "X`vxprint -g $failed_dg -F %use_type $movev`" = "Xswap" ] &&
					hr_conf_dumpdev $failed_dg $movev $failed_dm $cleardm
				vxplex -g $failed_dg -v $movev dis $movep
				vxedit -g $failed_dg -rf rm $movep
				echo "-" $movev $failed_dm $failed_dg >> $hr_rootlog

				set_to_env_locale
				(export movev movep failed_dm; egettxt "\
\\nRemirrored $movev volume plex $movep because of error on disk $failed_dm.
\\n" vxvmshm:971
				) | $MAILER -s "$_ms" "$recipients"
				set_to_c_locale
			fi

		# DRL/raid logs, or snap plexes
		elif [ X$movep = X`vxprint -Qp -g $failed_dg -e '
							pl_name=="'$movep'" &&
							(aslist->sd_is_log ||
							pl_state="SNAPDONE" || pl_state="SNAPATT" ||
							pl_state=="LOG" || pl_state=="BADLOG") ' \
							-F %name` ]; then
			logtype=""
			if [ X$movep = X`vxprint -Qp -g $failed_dg -e '
						pl_name=="'$movep'" &&
						(pl_state=="LOG" || pl_state=="BADLOG") ' \
						-F %name` ]; then
				logtype=raid
			elif [ X$movep = X`vxprint -Qp -g $failed_dg -e '
						pl_name=="'$movep'" && aslist->sd_is_log' \
						-F %name` ]; then
				logtype=drl
			else
				logtype=`echo \`vxprint -Qp -g $failed_dg -F %state $movep\`|
								sed -e "/^SNAP/!d"`
			fi
			[ "X$logtype" = "X" ] && logtype="-"

			move_ret=1
			hr_addlog_snapstart $failed_dg $movev $movep \
								"$failed_dm" "$cleardm" ""
			if [ $? -eq 0 ] ; then
				move_ret=0
				echo $movep $failed_dg $logtype >> $hr_rootlog
			fi

		# mirror volume plex
		elif [ X$movep = X`vxprint -Qp -g $failed_dg -e '
							pl_name=="'$movep'" &&
							(!aslist->sd_is_log &&
							pl_state!="LOG" && pl_state!="BADLOG") ' \
							-F %name` ]; then

			# move all the sds, even the good ones

            # relocate failed sds

			relc_ret=0
			relcsds=`vxprint -Qs -g $failed_dg -e 'assoc.pl_name="'$movep'" &&
					assoc.assoc.v_name="'$movev'" &&
					sd_dmname="'$failed_dm'" &&
					(sd_relocate || sd_nodevice || sd_kdetach)' -F %name`
			if [ "X$relcsds" != "X" ]; then

				vxassist -r -g $failed_dg move $movev \
						!$failed_dm $cleardm > $hr_tmp1 2> /dev/null
				relc_ret=$?

				if  [ $relc_ret -eq 0 ] && [ -s $hr_tmp1 ]; then
					cat $hr_tmp1 | sed -e "s/^/$movev /" > $hr_tmp2
					cat $hr_tmp2 >> $resultfile
					echo $failed_dg $movev >> $recoverlist
					echo "-" $movev $failed_dm $failed_dg >> $hr_rootlog
				fi
			fi

			# move good sds

			move_ret=0
			goodsds=`vxprint -Qs -g $failed_dg -e 'assoc.pl_name="'$movep'" &&
					assoc.assoc.v_name="'$movev'" &&
					sd_dmname="'$failed_dm'" &&
					(!sd_relocate && !sd_nodevice && !sd_kdetach)' -F %name`
			if [ "X$goodsds" != "X" ]; then

				hr_set_unreloc_flag "$failed_dg" "-m" "" hr_unreloc_flag

				vxassist $hr_unreloc_flag -g $failed_dg move $movev \
						!$failed_dm $cleardm 2>/dev/null
				move_ret=$?
			fi
		fi

		if [ $move_ret -eq 0 -a $relc_ret -eq 0 ]; then

			if [ X`cat $hr_rtdone | awk '$1=="'$movev'" {print $1}'` = "X" ]
			then
				echo $movev >> $hr_rtdone
				hr_put_hist $cleardm
			fi
		else
			if [ X`cat $hr_rtfail | awk '$1=="'$movev'" {print $1}'` = "X" ]
			then
				echo $movev >> $hr_rtfail
			fi
		fi

	done

	# create partitions for everything else
	/etc/vx/bin/vxbootsetup

	set_to_env_locale

	(export failed_dm failed_dg cleardm hr_rtdone hr_rtfail ; egettxt "\
\\nRootdisk $failed_dm of disk group $failed_dg experienced a failure,
hot-relocation attempted to move/remirror all the associated
subdisks to $cleardm
Subdisks from the following volumes got moved/remirrored:
\\n" vxvmshm:974
cat $hr_rtdone

	if [ -s $hr_rtfail ]; then
		egettxt "\
\\nError: Subdisks from the following volumes failed to move/remirror:
\\n" vxvmshm:970
cat $hr_rtfail
	fi
	) | $MAILER -s "$rootdm_ms" "$recipients"

	set_to_c_locale

	[ -s $hr_rtfail ] && return 13

	return 0
}

# create a new if plex is a log or a snapdone
hr_addlog_snapstart()
{
	_fdiskg=$1
	_fvol=$2
	_fplex=$3
	_fdm=$4
	_tdm=$5
	_sopt=$6

	_nfdm="!$_fdm"
	[ "X$_fdm" = "X" ] && _nfdm=""

	# if failed plex is a log
	if [ X$_fplex = X`vxprint -Qp -g $_fdiskg -e ' pl_name=="'$_fplex'" &&
					(aslist->sd_is_log ||
					pl_state=="LOG" || pl_state=="BADLOG") ' -F %name` ]
	then
		hr_set_unreloc_flag "$_fdiskg" "-l" "$_fplex" hr_unreloc_flag

		vxassist -g $_fdiskg $hr_unreloc_flag addlog $_fvol \
					$_sopt $_nfdm $_tdm > $vxassist_res 2>&1
		alret=$?

	# if failed plex is a snapdone
	elif [ X$_fplex = X`vxprint -Qp -g $_fdiskg -e ' pl_name=="'$_fplex'" &&
				(pl_state=="SNAPDONE" || pl_state=="SNAPATT") ' -F %name` ]
	then

		vxassist -g $_fdiskg snapstart $_fvol \
					$_sopt $_nfdm $_tdm > $vxassist_res 2>&1
		alret=$?
	fi

	grep ERROR $vxassist_res
	if [ $? -ne 0 -a $alret -eq 0 ] ; then
		vxplex -g $_fdiskg -o rm dis $_fplex
		hr_put_hist $_tdm
		return 0
	fi

	return 1
}

# The actual vxassist calls to relocate
hr_vxassist()
{
	targetdm=$1
	spareopt=$2

	# create new if plex is a log or a snap
	if [ X$failed_pl = X`vxprint -Qp -g $failed_dg -e '
			pl_name=="'$failed_pl'" &&
			(pl_state="SNAPDONE" || pl_state="SNAPATT" || aslist->sd_is_log ||
			pl_state=="LOG" || pl_state=="BADLOG")' -F %name` ]; then

		hr_addlog_snapstart $failed_dg $failed_v $failed_pl \
							"$failed_dm" "$targetdm" "$spareopt"
		return $?
	fi

	# not a log or a snap

	vxassist -r -g $failed_dg move $failed_v \
					$spareopt !$failed_dm $targetdm > $hr_tmp1 2> /dev/null

	if [ $? -eq 0 ] && [ -s $hr_tmp1 ]; then
		cat $hr_tmp1 | sed -e "s/^/$failed_v /" > $hr_tmp2
		cat $hr_tmp2 >> $resultfile
		hr_put_hist $targetdm
		return 0
	fi

	return 1
}

# attempt to hot-relocate with the provided pool of dms
hr_try_pool()
{
	diskpool=$1

	# select disks with enough space
	hr_select_spacedm $faillen 0 0 $diskpool $hr_fitdms

	# select a disk with no conflicts
	cleardm=`hr_get_cleardm "$hr_failvp" "$hr_fitdms" `

	# save current configuration
	set_to_env_locale
	hr_save_config
	set_to_c_locale

	if [ "X$cleardm" != "X" ]; then
		hr_vxassist $cleardm ""
		[ $? -eq 0 ] && return 0
	fi

	# a single disk cannot be found, try all disks in the pool.
	cat $diskpool | \
	while read trydm; do
		[ "X$trydm" = "X$cleardm" ] && continue
		hr_vxassist $trydm ""
		[ $? -eq 0 ] && return 0
	done

	# disregarding sd co-location
		a=`cat $diskpool | sed -n '1p' `
		spareflag=`vxprint -g $failed_dg -Qqd -F %spare $a`
		if [ "X$spareflag" = "Xon" ] ; then
			spareopt="spare=yes spare=only"
		else
			spareopt="spare=yes"
			hr_list_avail_disks off  on $hr_tmp4
			if [ -s $hr_tmp4 ] ; then
				for a in `cat $hr_tmp4` ; do
 					spareopt="$spareopt !$a"
 				done
			fi
		fi
		hr_vxassist "" "$spareopt"
		vxassist_ret=$?

	return $vxassist_ret
}

# an error occured on a disk that is not a rootdisk. Try to move first to 
# spare space and then free space.
hr_move_nonrootdm()
{
	# Sum all failed subdisks and list their volumes
	> $hr_failvp
	faillen=`vxprint -g $failed_dg -Qqse 'sd_dmname="'$failed_dm'"  &&
					(assoc.pl_kstate==DETACHED ||
					assoc.pl_nodarec || assoc.pl_recover ||
					sd_kdetach || sd_nodevice || sd_relocate) ' \
					-F '%len %v_name %pl_name' | \
					awk '{print $2 " " $3 >> "'$hr_failvp'" ; sum+=$1}
						END {printf "%d", 0+sum}' `
	sort < $hr_failvp | sed -e '/^$/d' > $hr_tmp1
	uniq < $hr_tmp1 > $hr_failvp
	[ "X$faillen" = "X" -o $faillen -eq 0 ] && return 0

	# get list of spares and try to use those

	hr_list_avail_disks on  off $hr_sparedms
	> $hr_nofuodms

	if [ -s $hr_sparedms ]; then
		# get list of spares from history
	    > $hr_tmp1
		histdm=`cat $hr_hist | awk '$1=="'$failed_dm'" && \
									$3=="'$failed_dg'" {print $2}'`
		if [ "X$histdm" != "X" ]; then
			vxprint -g $failed_dg -Qqd -F '%name %spare' $histdm |\
				awk '$2=="on" {print $1 >> "'$hr_tmp1'" }'
		fi
		cat $hr_sparedms >> $hr_tmp1
		cat $hr_tmp1 > $hr_sparedms

		hr_try_pool $hr_sparedms
		[ $? -eq 0 ] && return 0
	fi

	# spare space can't be used, get list of free disks not
	# marked with nohotuse.

	# check if spare=only flag is set as default
	if [ -s $VOL_DEFAULTS_FILE ]; then
		dsparefl=`cat $VOL_DEFAULTS_FILE | awk '{FS="#"; print $1}' | \
						awk ' {for (i=1; i<=NF; i++)
							if ($i ~ /^spare=/) { svar=$i }
							} END {print svar} ' `
	fi

	# non-spares can be used if not marked with nohotuse flag
	if [ "X$dsparefl" != "Xspare=only" ]; then
		hr_list_avail_disks off  off $hr_nofuodms

		if [ -s $hr_nofuodms ]; then
			# get list of non-spares from history
			> $hr_tmp1
			histdm=`cat $hr_hist | awk '$1=="'$failed_dm'" && \
										$3=="'$failed_dg'" {print $2}'`
			if [ "X$histdm" != "X" ]; then
				vxprint -g $failed_dg -Qqd -F '%name %spare' $histdm |\
					awk '$2=="off" {print $1 >> "'$hr_tmp1'" }'
			fi
			cat $hr_nofuodms >> $hr_tmp1
			cat $hr_tmp1 > $hr_nofuodms

			hr_try_pool $hr_nofuodms
			[ $? -eq 0 ] && return 0
		fi
	fi

	return 1
}

# hr_hot_relocate(failed_dm,failed_dg,failed_v)
# if root volume is on the failed disk, then all on that disk will be moved.
# else, only the failed sds will be moved.
# ret: errors larger than 10 indicates an error occurred
# while relocating the rootdisk.
# kept in the rootlog: successful log and data relocation,
# status of disk relocation, and whether notification was sent.

hr_hot_relocate()
{

	failed_dm=$1
	failed_dg=$2
	failed_v=$3

	> $vxassist_res

	# skip if already done.
	[ X`vxprint -g $failed_dg -ve 'aslist.aslist.sd_dmname="'$failed_dm'" &&
		v_name="'$failed_v'" ' -F %v_name ` = "X" ] && return 0

	# if the failed disk was the rootdisk, this volume was moved. if not
	# the rootdisk was partially moved (error #13), check log if the volume
	# was successfully relocated.

	rootdm_ret=`cat $hr_rootlog | awk '$1=="--" && $2=="'$failed_dm'" && \
										$3=="'$failed_dg'"	{ print $4 }'`
	if [ "X$rootdm_ret" != "X" ] ; then
		if [ $rootdm_ret -eq 13 ]; then
			relocres=`cat $hr_rootlog | \
				awk '$1=="-" && $2=="'$failed_v'" && $3=="'$failed_dm'" && \
							$4=="'$failed_dg'"	{ print $2 }'`
			[ "X$relocres" != "X" ] && return 0
		fi
		return $rootdm_ret
	fi

	rootplex=`vxprint -g $failed_dg -Qqpe 'assoc.v_use_type="root" && \
							aslist.sd_dmname="'$failed_dm'" ' -F %name`

	if [ "X$rootplex" != "X" ]; then
		hr_move_rootdm
		rootdm_ret=$?
		echo "--" $failed_dm $failed_dg $rootdm_ret >> $hr_rootlog

		# if there was an error while relocating the rootdisk,
		# check if the failed plex was relocated successfully
		if [ $rootdm_ret -eq 13 ]; then
			logtype=`cat $hr_rootlog | \
				awk '$1=="'$failed_pl'" && $2=="'$failed_dg'" { print $3 }'`
			relocres=`cat $hr_rootlog | \
				awk '$1=="-" && $2=="'$failed_v'" && $3=="'$failed_dm'" && \
							$4=="'$failed_dg'"	{ print $2 }'`
			if [ "X$logtype" != "X" ] || [ "X$relocres" != "X" ]; then
				return 0
			fi
		fi
		return $rootdm_ret

	fi

	# not rootdisk, move failed sds only
	hr_move_nonrootdm
	return $?
}

# add a new log plex
hr_create_new_plex()
{
	> $vxassist_res
	goodpl_name=`vxprint -Qp -g $dg_name -e 'pl_state!="LOG" 
					&& pl_state !~ /SNAP/
					&& pl_v_name=="'$v_name'" && pl_kstate==ENABLED &&
					!pl_log' -F %name`
	[ "X$goodpl_name" = "X" ] && return 1

	# using the previous version of hot-relocation
	if [ "X$oldhot" = "Xpre3.1" ]; then
		hr_addlog_snapstart $dg_name $v_name $pl_name "" "" "spare=yes"
		return $?
	fi

	# use the new version
	failed_pl=$pl_name
	logdms=`vxprint -g $dg_name -se 'assoc.pl_name="'$failed_pl'"' -F %dmname`

	# check if part of the log plex is on the rootdisk
	logonroot=""
	for failed_dm in $logdms; do
		rootplex=`vxprint -g $dg_name -Qqpe 'assoc.v_use_type="root" && \
						aslist.sd_dmname="'$failed_dm'" ' -F %name`
		[ "X$rootplex" != "X" ]	&& logonroot=$failed_dm
	done

	failed_dm=`echo $logdms | awk '{print $1}' `

	# set the failed dm to rootdisk if any part of the log plex is on it
	[ "X$logonroot" != "X" ] && failed_dm=$logonroot

	hr_hot_relocate $failed_dm $dg_name $v_name
	return $?
}

#
# try_reloc(dm, dg, volume)  attempt hot-relocation of subdisks
#
#
try_reloc()
{
	[ -n "$verbose" ] && echo "connect_spare $*" >&2

	dm_name=$1
	dg_name=$2
	v_name=$3

	_ms="Attempting VxVM relocation on host `uname -n`"

	# using the previous version of hot-relocation?
	if [ "X$oldhot" = "Xpre3.1" ]; then
		# If it is the root volume, do not relocate, create another
		# mirror
		if [ X$v_name = X`vxprint -AQve '(v_use_type="root" ||
				v_use_type="swap") && v_name="'$v_name'"' -F %name` ]; then
			_old_dms=`vxprint -g $dg_name -se '
							assoc.assoc.v_name="'$v_name'"' -F %dm_name `

			plexname=`vxprint -qQp -g $dg_name -e '(pl_kstate==DETACHED ||
						pl_nodarec) && pl_volume="'$v_name'"' -F %name`

			hr_set_unreloc_flag "$dg_name" "-l" "$plexname" hr_unreloc_flag

			vxassist -g $dg_name -o nofmr $hr_unreloc_flag mirror \
					$v_name spare=yes layout=nospan,contig
			vxassist_ret=$?
			if [ $vxassist_ret -eq 0 ] ; then

				# run vxbootsetup to create hard partitions for
				# swap and root
				/etc/vx/bin/vxbootsetup
				# reconfig dump device
				if [ "X`vxprint -g $dg_name -F %use_type $v_name`" = "Xswap" ]
				then
					_new_dms=`vxprint -g $dg_name -se '
								assoc.assoc.v_name="'$v_name'"' -F %dm_name `
					swplap=
					_new_swap_dm=
					for iswpdm in $_new_dms; do
						swplap=`echo $_old_dms | awk ' {for (i=1; i<=NF; i++)
											if ($i=="'$iswpdm'") {print $i}}'`
						[ "X$swplap" = "X" ] && _new_swap_dm=$iswpdm
					done
					[ -n "$_new_swap_dm" ] &&
					hr_conf_dumpdev  $dg_name $v_name $dm_name $_new_swap_dm
				fi

				vxplex -g $dg_name -v $v_name dis $plexname 
				vxedit -g $dg_name -rf rm $plexname
				set_to_env_locale
				(export v_name plexname dm_name; egettxt "\
\\nRemirrored $v_name volume plex $plexname because of error on disk $dm_name.
\\n" vxvmshm:584
				) | $MAILER -s "$_ms" "$recipients"
				set_to_c_locale
				return 0
   	        fi
		else

			vxassist -r -g $dg_name move $v_name !$dm_name \
				spare=yes > $hr_tmp1
			vxassist_ret=$?
			if [ $vxassist_ret -eq 0 ]; then
				if [ -s $hr_tmp1 ] ; then
					cat $hr_tmp1 | sed -e "s/^/$v_name /" > $hr_tmp2
					cat $hr_tmp2 >> $resultfile
				fi
				return 0
			fi
		fi
	else
		# relocate the failed sds with the new version of hot-relocation
		failed_pl="-"
		hr_hot_relocate $dm_name $dg_name $v_name
		vxassist_ret=$?

		[ $vxassist_ret -eq 0 ] && return 0

		# If errors while moving from the rootdisk, check if the notice below
		# was sent for this disk.
		if [ $vxassist_ret -gt 10 ]; then
			[ X`cat $hr_rootlog | awk '$1=="---" && $2=="'$failed_dm'" && \
				$3=="'$failed_dg'" {print $2}'` != "X" ] &&
					return $vxassist_ret
			echo "---" $failed_dm $failed_dg >> $hr_rootlog
		fi
	fi


	# If we've gotten here, relocation was not successful,
	# so send mail
   	(
		set_to_env_locale
		export v_name dm_name dg_name; egettxt \
"Relocation was not successful for subdisks on disk $dm_name in
volume $v_name in disk group $dg_name.  No replacement was made and the
disk is still unusable." vxvmshm:585

		avols=`affvols $dm_name $dg_name`
		dvols=`deadvols $dm_name $dg_name`
		draid=`deadraids $dm_name $dg_name`
		alldead="${dvols}${dvols:+ }${draid}"

		if [ ! -z "$avols" ] 
		then
			export avols dm_name; egettxt "
The following volumes have storage on ${dm_name}:

$avols

These volumes are still usable, but the the redundancy of
those volumes is reduced. Any RAID-5 volumes with storage on 
the failed disk may become unusable in the face of further 
failures." vxvmshm:535
		fi

		if [ ! -z "$dvols" ]
		then
			 export dvols dm_name; egettxt "
The following volumes:

$dvols

have data on $dm_name but have no other usable mirrors on other
disks. These volumes are now unusable and the data on them is
unavailable. These volumes must have their data restored." vxvmshm:587

		fi

		if [ ! -z "$draid" ]
		then
			export draid dm_name; egettxt "

The following RAID-5 volumes:

$draid

have storage on $dm_name and have experienced other failures. These
RAID-5 volumes are now unusable and the data on them is unavailable.
These RAID-5 volumes must have their data restored." \
vxvmshm:625
		fi
	) | $MAILER -s "$_ms" "$recipients"

	set_to_c_locale

	return $vxassist_ret
}

# search_relocate_subdisk - search for subdisk marked for relocation based on a plex name

search_relocate_subdisk()
{
        DGNAME=$1
        PLNAME=$2
        vxprint -Qs -g $DGNAME -e '(sd_relocate || sd_nodevice || sd_kdetach)
                && sd_pl_name="'$PLNAME'"' -F %name >> $failsd_list
        for SDNAME in `vxprint -Qs -g $DGNAME -e 'sd_pl_name="'$PLNAME'"' -F %name`
        do
                DMNAME=`vxprint -g $DGNAME -m $SDNAME | grep "dm_name=" | awk -F= '{print $2}'`
                for PLNAME in `vxprint -Qp -g $DGNAME -e 'pl_v_name="'$DMNAME'"' -F %name`
                do
                        search_relocate_subdisk $DGNAME $PLNAME
                done
        done
}

hr_cleanup()
{
	rm -f $failsd_list $faildisks $plexlist $reloc_dm_list
	rm -f $raidlist $vxassist_res $vxrecover_res $hr_newvp
	rm -f $hr_failvp $hr_sparedms $hr_nofuodms $hr_fitdms
	rm -f $hr_tmp1 $hr_tmp2 $hr_rootlog $hr_rtdone $hr_rtfail
	rm -f $hr_tmp3 $hr_tmp4 $tmpfile $tmpfile2
}

# Checkdetach - Look for failed objects and send the administrator mail

checkdetach()
{
	hr_hist=$hrhistfile				# list of source-target-group relocations
	hr_newvp=/tmp/hreloc-2t$$		# vols and plexes on a good dm
	hr_failvp=/tmp/hreloc-3t$$		# vols and plexes on a failed dm
	hr_tmp1=/tmp/hreloc-4t$$
	hr_sparedms=/tmp/hreloc-5t$$	# spare dms
	hr_nofuodms=/tmp/hreloc-6t$$	# !nohotuse dms
	hr_fitdms=/tmp/hreloc-7t$$		# dms with enough space
	hr_tmp2=/tmp/hreloc-8t$$
	hr_rootlog=/tmp/hreloc-9t$$		# hist of activities on rootdisk
	hr_rtdone=/tmp/hreloc-bt$$		# list of sucessful moves from rootdisk
	hr_rtfail=/tmp/hreloc-ct$$		# list of failed moves from rootdisk
	hr_tmp3=/tmp/hreloc-dt$$
	hr_tmp4=/tmp/hreloc-et$$

	# snapshot of configuration before hot-relocation is attempted.
	failed_dir=$VOL_SAVECONFIG_DIR
	failed_file=`date '+%y%m%d_%H'``date '+%M'``date '+%S'`.mpvsh

	faildisks=/tmp/hs-4$$		# list of failed disks
	plexlist=/tmp/hs-5$$		# list of detached plexes
	reloc_dm_list=/tmp/hs-6$$	# list of dm's from which to relocate
	raidlist=/tmp/hs-7$$		# list of failed raid disks
	recoverlist=$hrrecoverfile	# list of volumes to recover
	failsd_list=/tmp/sd-1$$         # list of failed subdisks
	vxassist_res=/tmp/hs-vxas$$     # vxassist output
	vxrecover_res=/tmp/hs-rec$$     # vxrecover otuput
	tmpfile=/tmp/hs-9$$
	tmpfile2=/tmp/hs-t$$
	resultfile=$hrresultfile
	
	rm -f $faildisks
	[ ! -f $hr_hist ] && > $hr_hist
	[ ! -f $resultfile ] && > $resultfile
	> $hr_rootlog

	trap 'hr_cleanup ' 1 2 3 15
 
	_ms="Volume Manager failures on host `uname -n`"

	vxprint -AQdF '%name %nodarec %dgname' 2> /dev/null | \
		awk '$2=="on" {print " " $1 " " $3}' > $faildisks
	d=`awk '{print " " $1}' < $faildisks`

	# Create a list of failed raid subdisks and their dg and volume
	# Then check each volume.  If it is not detached we can relocate
	# the failed subdisk, so add it to the list to relocate.  Otherwise,
	# mail an error message.

	rm -f $reloc_dm_list $raidlist
	vxprint -AQse 'assoc->pl_layout=RAID  
		&& (sd_kdetach || sd_nodevice || sd_relocate)
		&& assoc->assoc->v_kstate=ENABLED' -F '%name %dg_name'\
		| sed -e '/^$/d' >$tmpfile
	cat $tmpfile | \
	while read sd_name dg_name
	do
		dm_name=`vxprint -g $dg_name -F "%dm_name" $sd_name`
		v_name=`vxprint -g $dg_name -F "%v_name" $sd_name`
		echo $dm_name $dg_name $v_name >>$reloc_dm_list
	done
	
	  # create a list of disabled raid volumes
	 vxprint -AQve 'aslist->pl_layout=RAID
                && (aslist->aslist->sd_kdetach ||
                    aslist->aslist->sd_nodevice ||
                    aslist->aslist->sd_relocate)
		&& v_kstate!=ENABLED' -F %name | sed -e '/^$/d' >$tmpfile
	 cat $tmpfile | \
         while read vol_name
         do
	         (export vol_name; egettxt "\
\\nRaid volume $vol_name experienced a failure, but no relocation
will be done as the volume has been detached or disabled 
\\n" vxvmshm:588
                ) | $MAILER -s "$_ms" "$recipients"
         done

	# create a list of detached plexes (or plexes with no da record)
	# and their dg and volume
	vxprint -AQpe 'pl_kstate==DETACHED || pl_nodarec || pl_recover' \
		-F '%name %dgname %vname' 2>/dev/null | \
		sed -e '/^$/d' > $plexlist

	# set failing flag on dm if plex is log
	cat $plexlist | \
 	while read pl_name dg_name v_name; do
	    [ X$v_name = "X-" ] && continue

		islogplex=`vxprint -Qp -g $dg_name -e 'pl_name=="'$pl_name'"
					&& (aslist->sd_is_log || pl_state="BADLOG" ||
				     pl_state="LOG")' -F %name`
		if [ "X$islogplex" = "X$pl_name" ]; then
			a=`vxprint -g $dg_name -se 'assoc.pl_name="'$pl_name'"' \
						-F %dmname 2> /dev/null `
			for b in $a; do
				[ X`vxprint -g $dg_name -Qqd -F %nodarec $b` != "Xon" ] &&
					vxedit -g $dg_name set failing=on $b
			done
		fi
	done

	# create a list of bad plexes then send mail about failed disks
	# and bad plexes
	p=`vxprint -AQpe '(pl_kstate==DETACHED || pl_recover ||
		pl_nodarec) && (pl_state!="LOG" && pl_state!="BADLOG" &&
		!aslist->sd_is_log)' -F %name 2>/dev/null`
	
	lp=`vxprint -AQpe '(pl_kstate==DETACHED || pl_nodarec || pl_recover)
		&& (pl_state=="LOG" || pl_state=="BADLOG" || aslist->sd_is_log)' -F %name`

	failing_d=`vxprint -AQdF '%name %failing' | awk '$2=="on" {print $1}'`

	if [ ! -z "$d" ] || [ ! -z "$p" ]  || [ ! -z "$lp" ] || [ ! -z "$failing_d" ]
	then
		set_to_env_locale
		( egettxt "\
Failures have been detected by the VERITAS Volume Manager:" \
vxvmshm:191
		[ -z "$d" ] || \
			d="$d" egettxt "\\nfailed disks:\\n$d" vxvmshm:520
		[ -z "$p" ] || \
			p="$p" egettxt "\\nfailed plexes:\\n$p" vxvmshm:521
		[ -z "$lp" ] || \
			lp="$lp" egettxt "\\nfailed log plexes:\\n$lp" vxvmshm:590
		[ -z "$failing_d" ] || \
			export failing_d; egettxt "\\nfailing disks:\\n$failing_d" vxvmshm:591
		[ -n "$d" ] && egettxt "\

The Volume Manager will attempt to find spare disks, relocate failed
subdisks and then recover the data in the failed plexes. 
" vxvmshm:624
		) | $MAILER -s "$_ms" "$recipients"
		set_to_c_locale
	fi

	# check mirror vol plexes
	cat $plexlist | \
 	while read pl_name dg_name v_name; do
	    [ X$v_name = "X-" ] && continue

		if [ X$pl_name = X`vxprint -Qp -g $dg_name -e 'pl_name=="'$pl_name'" 
				&& (!aslist->sd_is_log && pl_state!="BADLOG"
				&& pl_state!="SNAPDONE" && pl_state!="SNAPATT"
			    && pl_state!="LOG")' -F %name` ] ; then

			goodpl_name=`vxprint -Qp -g $dg_name -e 'pl_state!="LOG" 
					&& pl_state !~ /SNAP/
					&& pl_v_name=="'$v_name'" && pl_kstate==ENABLED &&
					!pl_log' -F %name`
			rm $failsd_list > /dev/null 2>&1
			search_relocate_subdisk $dg_name $pl_name

			# A disk failure will not mark the subdisks with a relocate
			# but it is disabled, we should move them too.
			cat $failsd_list | \
			while read failsd_name
			do

			# If a failed plex has a good mirror, save the name of the
		    	# subdisk, dg, and volume, so it can be relocated.

			    if [ ! -z "$goodpl_name" ] 
			    then 

				dm_name=`vxprint -g $dg_name -F '%dm_name' $failsd_name`
				echo $dm_name $dg_name $v_name >>$reloc_dm_list
				offset=`vxprint -g $dg_name -F '%dev_offset' $failsd_name`
				len=`vxprint -g $dg_name -F '%len' $failsd_name`
				da_name=`vxprint -g $dg_name -F %last_da_name $dm_name`
				set_to_env_locale
				( export failsd_name pl_name goodpl_name offset len dm_name da_name; egettxt "\
\\nAttempting to relocate subdisk $failsd_name from plex $pl_name.
Dev_offset $offset length $len dm_name $dm_name da_name $da_name.
The available plex $goodpl_name will be used recover the data. 
\\n" vxvmshm:597
				) | $MAILER -s "$_ms" "$recipients"
				set_to_c_locale
			    else
				set_to_env_locale

				( export pl_name; egettxt "\
\\nUnable to relocate failed subdisk from plex $pl_name because no
suitable mirror was found from which to recover data. 
\\n" vxvmshm:598
				) | $MAILER -s "$_ms" "$recipients"
				set_to_c_locale
			    fi 
			done
		fi
	done

	# check logs

	cat $plexlist | \
 	while read pl_name dg_name v_name
	do
	    # skip this plex if it isn't associated
	    [ X$v_name = "X-" ] && continue

		# check if plex was on the rootdisk's log; if yes, it was relocated.
		logtype=""
		if [ X`vxprint -g $dg_name -pe 'pl_name="'$pl_name'" ' \
				-F %name ` = "X" ]; then
			logtype=`cat $hr_rootlog | \
					awk '$1=="'$pl_name'" && $2=="'$dg_name'" { print $3 }'`
		fi

		# snap plexes cannot be moved and get relocated like log plexes
		snapstate=`vxprint -Qp -g $dg_name -F %state $pl_name 2> /dev/null`
		if [ "X$snapstate" = "XSNAPDONE" -o "X$snapstate" = "XSNAPATT" ] ||
			[ "X$logtype" = "XSNAPDONE" -o "X$logtype" = "XSNAPATT" ]; then

			[ "X$snapstate" = "X" ] && snapstate=$logtype
			[ "X$logtype" = "X" ] && hr_create_new_plex

			if [ $? -eq 0 ] || [ "X$logtype" != "X" ]; then 
			set_to_env_locale
			(export snapstate pl_name v_name; egettxt "\
\\nA $snapstate plex $pl_name in volume $v_name failed.
A new plex for volume $v_name was created.
\\n" vxvmshm:969
               		) | $MAILER -s "$_ms" "$recipients"
			set_to_c_locale
			else 
			set_to_env_locale
			(export vxassist_res snapstate pl_name v_name; egettxt "\
\\nA $snapstate plex $pl_name failed, 
creation of new $snapstate plex for volume $v_name failed.
\\n" vxvmshm:968
cat $vxassist_res
			) | $MAILER -s "$_ms" "$recipients"
			set_to_c_locale
			fi 
		
	    # Is this a raid log plex

            elif [ X$pl_name = X`vxprint -Qp -g $dg_name -e '
				(pl_state=="LOG" || pl_state=="BADLOG") &&
				pl_name=="'$pl_name'"' -F %name` ] || [ "X$logtype" = "Xraid" ]
	    then 

		[ "X$logtype" = "X" ] && hr_create_new_plex

		if [ $? -eq 0 ] || [ "X$logtype" != "X" ]; then 
			set_to_env_locale
			(export pl_name v_name; egettxt "\
\\nRaid log $pl_name in volume $v_name failed.
A new raid log for volume $v_name was created.
\\n" vxvmshm:592
               		) | $MAILER -s "$_ms" "$recipients"
			set_to_c_locale
		else 
			set_to_env_locale
			(export vxassist_res pl_name v_name; egettxt "\
\\nRaid log $pl_name failed, 
creation of new raid log for volume $v_name failed.
\\n" vxvmshm:594
cat $vxassist_res
			) | $MAILER -s "$_ms" "$recipients"
			set_to_c_locale
		fi 

		            # is this a DRL plex 
	    elif [ X$pl_name = X`vxprint -Qp -g $dg_name -e '
				pl_name=="'$pl_name'" &&
				aslist->sd_is_log' -F %name` ] || [ "X$logtype" = "Xdrl" ]
		then 		

		set_to_env_locale
		[ "X$logtype" = "X" ] && hr_create_new_plex

		if [ $? -eq 0 ] || [ "X$logtype" != "X" ]; then 
			(export pl_name v_name; egettxt "\
\\nRemoved detached DRL plex $pl_name, 
a new DRL log for volume $v_name was created.
\\n" vxvmshm:595
	                ) | $MAILER -s "$_ms" "$recipients"
                 else 
			(export vxassist_res pl_name v_name; egettxt "\
\\nDRL $pl_name detached, Creating new DRL for volume $v_name failed.
\\n" vxvmshm:596; cat $vxassist_res
			) | $MAILER -s "$_ms" "$recipients"
                 fi
		set_to_c_locale

	      fi  
		
	done
	
		# attempt relocation of all dms in the list
		if [ -s $reloc_dm_list ]
		then
			# remove dups if multiple subdisks on dm failed
			sort <$reloc_dm_list >$tmpfile
			uniq <$tmpfile >$reloc_dm_list
			cat $reloc_dm_list | \
			while read dm_name dg_name v_name
			do
				vxdg list | grep $dg_name|grep shared >/dev/null
				if [ $? -eq 0 ]; then
					# Only master will start the relocation
					vxdctl -c mode | grep SLAVE > /dev/null
					if [ $? -eq 0 ]; then
					set_to_env_locale
( export dg_name; egettxt "hot-relocation operation for shared disk group $dg_name will be performed on master node" vxvmshm:863
) | $MAILER -s "$_ms" "$recipients"
					set_to_c_locale
						continue
					fi
				fi

				# if relocation works then add volume to
				# list of volumes to recover
				try_reloc $dm_name $dg_name $v_name"$@"
				[ $? -eq 0 ] && echo $dg_name $v_name >> $recoverlist

				if [ -s $resultfile ]
				then
					set_to_env_locale
					(
					cat $resultfile | \
					while read r_v_name a b sd_name c newsd_name
					do
						export sd_name newsd_name r_v_name; egettxt "\
Volume $r_v_name Subdisk $sd_name relocated to $newsd_name,
but not yet recovered.\\n" vxvmshm:962
					done
					) | $MAILER -s "$_ms" "$recipients"
					set_to_c_locale
				fi
				> $resultfile
			done
		fi
                if [ -s $recoverlist ]
                then
			sort <$recoverlist >$tmpfile
			uniq <$tmpfile >$recoverlist
                        cat $recoverlist | \
                        while read dg_name v_name
                        do

		# recover snapdis plexes.
		snapplex=`vxprint -g $dg_name -pe ' assoc.v_name="'$v_name'" &&
						(pl_state="SNAPDIS" || pl_state="SNAPTMP") &&
						pl_kstate!=ENABLED' -F %name `
		snapret=0
		if [ "X$snapplex" != "X" ]; then
			for a in $snapplex; do
				vxplex -g $dg_name dis $a
				vxplex -g $dg_name snapstart $v_name $a > $vxrecover_res 2>&1
				snapret=$?
				grep ERROR $vxrecover_res
				[ $? -eq 0 ] && snapret=1
			done
		fi

                                if [ -n "$o_args" ]
				then
				    vxrecover -g $dg_name -o $o_args -s $v_name   > $vxrecover_res 2>&1 
				    ret=$?
				    grep ERROR $vxrecover_res
				else
				    vxrecover -g $dg_name -s $v_name  > $vxrecover_res 2>&1 
				    ret=$?
				    grep ERROR $vxrecover_res
				fi

				if [ $ret -ne 0 -o $? -ne 1 -o $snapret -ne 0 ]
				then
					set_to_env_locale
					(export dg_name v_name vxrecover_res; egettxt "\
Failure recovering $v_name in disk group $dg_name.\n" vxvmshm:600
cat $vxrecover_res
					) | $MAILER -s "$_ms" "$recipients"
					set_to_c_locale
				else
					set_to_env_locale
					(export dg_name v_name vxrecover_res; egettxt "\
Recovery complete for volume $v_name in disk group $dg_name.\n" vxvmshm:601 \
					) | $MAILER -s "$_ms" "$recipients"
					set_to_c_locale
                                fi
                        done
		fi

		rm -f $hr_hist $recoverlist $resultfile
		hr_cleanup
}
 
checkfailing()
{
	failing_d1=`vxprint -AQdF '%name %failing %nodarec' | 
			awk '$2=="on" && $3=="off" {print $1}'`
	if [ -n "$failing_d1" ]
	then
		for i in $failing_d1
		do
			vxdisk check $i 2>/dev/null
		done

		checkfailingcnt=`expr $checkfailingcnt + 1`

		failing_d1=`vxprint -AQdF '%name %failing %nodarec' | 
				awk '$2=="on" && $3=="off" {print $1}'`
		if [ -n "$failing_d1" ] &&  [ $checkfailingcnt -lt 4 ]; then
			docheckfailing=yes
		else
			docheckfailing=no
		fi
	fi
}
# Get events from vold, waiting for 15 seconds of quiescence before
# checking for failed objects.
 
docheck=yes
docheckfailing=yes
checkfailingcnt=0

vxnotify -f -w 15 | while read code more
do
	[ -z "$verbose" ] || echo "Code: $code"
	case $code in
	waiting)    if [ "$docheck" = yes ]
			then
				checkdetach "$@"
				docheck=no
			fi
			if [ "$docheckfailing" = yes ]
			then
				checkfailing "$@"
			fi;;
	log-detach|detach|relocate|more|connected)
		checkfailingcnt=0
		docheckfailing=yes
                docheck=yes;;
	esac

done

