path: root/lvm/sanlvm

#!/bin/bash

cmd=
glpath=
vgname=
lvname=
offset=
suffix=
sanlvmd=
host_id=0
lock_mode=
no_active=
active_mode=

# sanlvmd is only needed when lv activation locks are used.
# It is not needed for the vglk, which protects vg metadata
# by serializing commands changing the vg.
#
# sanlvmd is a process that sanlock views as owning the lv
# activation leases.  When a vg disappears/fails, sanlock
# will notify the owner of the leases in that vg; in this
# case, it sends SIGTERM to sanlvmd.  When sanlvmd gets
# this SIGTERM, it knows that its leases will be disappearing
# soon, and that another host would be able to acquire them.
# So, it needs to force the local host to stop using the
# lv's it has locked, since they won't be locked much longer.
#
# To do this, it could:
# - forcibly deactivate the sanlvm lvs in the failed vgs
# - suspend the lv's in the failed vgs
# - or suspend lv's, replace with the error target and resume
# and once done, it can release the leases on those lvs.
#
# (If the sanlock watchdog is enabled, it will fire to reset
# the host if sanlvmd is not able to release the leases
# before another host could acquire them.)
#
# If the lv's are suspended, sanlvmd could monitor and wait
# for the vg to reappear.  When it does, it could reacquire
# the leases for the suspended lv's and then resume the lv's.
# (Unless the leases have been acquired by another host since
# they were released, in which case they cannot be reacquired
# by the oringal host, and the lv's will not be resumed.)

help() {
	vg="\$vg"
	host_id="\$host_id"
	echo ""
	echo "/etc/sanlvm.conf"
	echo "   must contain unique host_id definition from 1 to 2000, e.g."
	echo "   host_id 1"
	echo ""
	echo "sanlvm vgcreate [options] $vg <devs>"
	echo "   create vg, create and initialize storage in $vg for sanlock to use"
	echo ""
	echo "sanlvm start"
	echo "   start lockspaces for sanlvm vgs, created above"
	echo ""
	echo "sanlvm <lvmcmd> ..."
	echo "   lvm commands within $vg are serialized by sanlock"
	echo ""
	echo "sanlvm [--na] lvcreate ..."
	echo "   create an lv with or without (--na) an associated lv"
	echo "   activation lease.  without an activation lease, the"
	echo "   the activation state among hosts is ignored."
	echo ""
	echo "sanlvm --sh|ex lvchange -ay ..."
	echo "   acquire a shared or exclusive lease, then activate the lv."
	echo "   an exclusive lease prevents more than one host from activating"
	echo "   the lv at once, a shared lease allows multiple hosts to"
	echo "   activate the lv at once, and prevents any from activating"
	echo "   it exclusively."
	echo ""
	echo "sanlvm lvchange -an ..."
	echo "   deactivate the lv, then release its lease"
	echo ""
	exit 0
}

# vg name is the arg preceding the first pv path arg,
# or the last arg when no pv paths exist
# e.g. vgcreate, vgchange, vgextend, lvcreate

get_vg_name() {
	prev=$2

	for i in $get_args
	do 
		first=`expr substr $i 1 1`
		if [[ "$first" == "/" ]]; then
			vgname=$prev
			return
		fi

		prev=$i
	done

	for i in $get_args
	do 
		vgname=$i
	done
}

# lv name follows -n or --name arg
# FIXME: this doesn't work without a space, i.e. -nfoo

get_lv_name() {
	save_next=0

	for i in $get_args
	do 
		if [[ $save_next == 1 ]]; then
			lvname=$i
			return
		fi

		if [[ "$i" == "-n" ]] || [[ "$i" == "--name" ]]; then
			save_next=1
			continue
		fi
	done
}

# vg name is part of an lv path, e.g. lvextend, lvremove
# first arg containing '/'

get_vg_lv_name() {
	for i in $get_args
	do 
		ind=`expr index $i /`
		if [ $ind -gt 0 ]; then
			lvname=`basename $i`
			dir=`dirname $i`
			vgname=`basename $dir`
			return
		fi
	done
}

get_host_id() {
	host_id=`awk '/^host_id/ { print $2 }' < /etc/sanlvm.conf`

	if [[ $? -ne 0 ]]; then
		echo "no host_id found in /etc/sanlvm.conf"
		exit 1
	fi
}

get_gl_path() {
	glpath=`awk '/^global_lease_path/ { print $2 }' < /etc/sanlvm.conf`

	if [[ $? -ne 0 ]]; then
		echo "no global_lease_path found in /etc/sanlvm.conf"
	fi
}

# search lvchange/vgchange args for -a, ---available
# if there's a "y" anywhere after the option, then active_mode="y"
# if there's a "n" anywhere after the option, then active_mode="n"

get_active_mode() {
	for i in $get_args
	do 
		if [[ "$i" == "-ay" ]] || [[ "$i" == "-aly" ]] || [[ "$i" == "-aey" ]]; then
			active_mode="y"
			return
		fi

		if [[ "$i" == "-an" ]] || [[ "$i" == "-aln" ]] || [[ "$i" == "-aen" ]]; then
			active_mode="n"
			return
		fi

		if [[ "$i" == "-a" ]] || [[ "$i" == "--available" ]]; then
			check_next=1
			continue
		fi

		if [[ $check_next == 1 ]]; then
			ind=`expr index $i y`
			if [ "$ind" -gt 0 ]; then
				active_mode="y"
			else
				active_mode="n"
			fi
			return
		fi
	done
}

# get the offset value from the @sanlvm_offset_xyz tag on the lv

get_lease_offset() {
	tags=`lvs --noheadings -o lv_name,lv_tags $vgname | grep $lvname | grep sanlvm_offset_ | awk '{print $2}'`

	if [[ "$tags" == "" ]]; then
		return
	fi

	# FIXME: handle case where the lv has tags in addition to ours

	offset=`echo $tags | cut --delimiter=_ --fields=3`
}

check_vg_start() {
	sanlock client status | grep 's $vgname'

	if [[ $? -eq 0 ]]; then
		# FIXME: just start the lockspace here?
		echo "lockspace for $vgname must be started"
		exit 1
	fi
}

vgcreate() {
	lvm $get_args

	if [[ $? -ne 0 ]]; then
		exit 1
	fi

	lvm lvcreate -n leases -L 1G $vgname

	if [[ $? -ne 0 ]]; then
		exit 1
	fi

	sanlock direct init -s $vgname:0:/dev/$vgname/leases:0

	if [[ $? -ne 0 ]]; then
		exit 1
	fi

	sanlock direct init -r $vgname:vglk:/dev/$vgname/leases:1048576

	if [[ $? -ne 0 ]]; then
		exit 1
	fi

	lvm vgchange --addtag @sanlvm $vgname

	if [[ $? -ne 0 ]]; then
		exit 1
	fi

	exit 0
}

lvchange() {
	if [[ "$active_mode" == "y" ]]; then

		sanlock client acquire -r $vgname:$lvname:/dev/$vgname/leases:$offset$suffix -p $sanlvmd

		if [[ $? -ne 0 ]]; then
			echo "lv lock failed for $vgname/$lvname, may be active on another host"
			return
		fi

		lvm lvchange -ay $vgname/$lvname
	else
		lvm lvchange -an $vgname/$lvname

		if [[ $? -ne 0 ]]; then
			echo "deactivate failed for $vgname/$lvname, may be open"
			return
		fi

		sanlock client release -r $vgname:$lvname:/dev/$vgname/leases:$offset -p $sanlvmd
	fi
}

if [[ $# == 0 ]]; then
	help
fi

arg1=$1

if [[ "$arg1" == "help" ]]; then
	help
fi

if [[ "$arg1" == "--ex" ]] || [[ "$arg1" == "--sh" ]]; then
	lock_mode=$arg1
	cmd=$2
elif [[ "$arg1" == "--na" ]]; then
	no_active=$arg1
	cmd=$2
else
	cmd=$1
fi

if [[ "$cmd" == "vgcreate" ]] || [[ "$cmd" == "vgremove" ]] || [[ "$cmd" == "vgextend" ]]; then
	get_gl_path

	if [[ "$glpath" == "" ]]; then
		echo "Warning: no global_lease_path found in /etc/sanlvm.conf"
		echo "Concurrent vgcreate, vgremove, and vgextend commands"
		echo "will not be properly synchronized among hosts."
		echo "(Concurrent vgextend of the same vg will be proteted,"
		echo "and operations within a vg will be protected.)"
	fi
fi

if [[ "$cmd" == "global_init" ]] || [[ "$cmd" == "global_init_force" ]]; then
	get_gl_path

	if [[ "$glpath" == "" ]]; then
		echo "no global_lease_path found in /etc/sanlvm.conf"
		exit 1
	fi

	sanlock client status | grep 's sanlvm_global'

	if [[ $? -eq 0 ]]; then
		echo "$glpath already initialized and in use."
		exit 1
	fi

	sanlock direct read_id -s sanlvm_global:1:$glpath:0

	if [[ $? -eq 0 ]] && [[ "$cmd" == "global_init" ]]; then
		echo "$glpath already initialized and may be in use."
		echo "use sanlvm global_init_force to override."
		exit 1
	fi

	sanlock direct init -s sanlvm_global:0:$glpath:0

	if [[ $? -ne 0 ]]; then
		exit 1
	fi

	sanlock direct init -r sanlvm_global:lease:$glpath:1048576

	if [[ $? -ne 0 ]]; then
		exit 1
	fi

	exit 0

elif [[ "$cmd" == "_start" ]]; then
	# wrapped by shared sanlvm_global:lease

	vgname=$SANLVM_VGNAME
	host_id=$SANLVM_HOST_ID

	# Parallel starts among hosts with SH lease are ok.
	# We don't want to start the vg/lockspace while another
	# host is removing it with vgremove.
	#
	# vgcreate/vgremove/vgextend use the exclusive mode.
	# We want to serialize this start with vgremove; vgremove
	# holds the ex global lease and checks that the lockspace/vg
	# is not added/in-use on any other hosts.

	lvm lvchange -a y /dev/$vgname/leases

	if [[ $? -ne 0 ]]; then
		echo "failed to activate /dev/$vgname/leases"
		exit 1
	fi

	echo "adding lockspace $vgname, this may take minutes..."
	sanlock add_lockspace -s $vgname:$host_id:/dev/$vgname/leases:0

	if [[ $? -ne 0 ]]; then
		exit 1
	fi

	exit 0

elif [[ "$cmd" == "start" ]]; then
	get_gl_path
	get_host_id

	if [[ "$glpath" != "" ]]; then
		sanlock client status | grep 's sanlvm_global'

		if [[ $? -eq 1 ]]; then
			echo "starting lockspace sanlvm_global, this may take minutes..."
			lvm lvchange -a y $glpath
			sanlock add_lockspace -s sanlvm_global:$host_id:$glpath:0
		fi
	fi

	for vgname in `vgs @sanlvm --noheadings -o vg_name`; do

		sanlock client status | grep 's $vgname'

		if [[ $? -eq 0 ]]; then
			continue
		fi

		# FIXME: do something to prevent sanlvm lv's from
		# automatically being activated by the system at
		# startup; then this vgchange -an is not needed.

		vgchange -an $vgname

		if [[ "$glpath" == "" ]]; then
			echo "no global lock"
			lvm lvchange -a y /dev/$vgname/leases
			echo "starting lockspace $vgname, this may take minutes..."
			sanlock add_lockspace -s $vgname:$host_id:/dev/$vgname/leases:0
		else
			export SANLVM_VGNAME=$vgname
			export SANLVM_HOST_ID=$host_id
			sanlock command -r sanlvm_global:lease:$glpath:1048576:SH -c ./sanlvm _start "$@"
		fi
	done

	exit 0

elif [[ "$cmd" == "stop" ]]; then
	get_gl_path
	get_host_id

	# rem_lockspace will fail if any locks exist in it, i.e.
	# if vglk is being used, or any lv's are activated with a lock

	for vgname in `vgs @sanlvm --noheadings -o vg_name`; do
		sanlock rem_lockspace -s $vgname:$host_id:/dev/$vgname/leases:0

		if [[ $? -eq 0 ]]; then
			lvm lvchange -an $vgname/leases
		else
			echo "cannot stop lockspace $vgname"
		fi
	done

	sanlock rem_lockspace -s sanlvm_global:$host_id:$glpath:0

	exit 0

elif [[ "$cmd" == "_vgcreate" ]]; then
	# wrapped by sanlvm_global:lease
	shift

	get_args=$*
	vgname=$SANLVM_VGNAME

	vgcreate

elif [[ "$cmd" == "vgcreate" ]]; then

	get_args=$*
	get_vg_name

	if [[ "$glpath" == "" ]]; then
		echo "no global lock"
		vgcreate
	else
		export SANLVM_VGNAME=$vgname
		sanlock command -r sanlvm_global:lease:$glpath:1048576 -c ./sanlvm _vgcreate "$@"
	fi

	if [[ $? -ne 0 ]]; then
		exit 1
	fi
	
	exit 0

elif [[ "$cmd" == "_vgremove" ]]; then
	# wrapped by sanlvm_global:lease
	shift

	vgname=$SANLVM_VGNAME

	get_host_id

	others=`sanlock client host_status -s $vgname | egrep -v -e ^$host_id -e 'timestamp 0'`

	# NB this check can be inaccurate for a short time.
	# after another host stops, it takes until our next
	# renewal for us to notice and show timestamp 0.

	if [[ $? -eq 0 ]]; then
		echo "cannot remove vg, other hosts using vg $vgname require sanlvm stop:"
		echo "$others"
		exit 1
	fi

	sanlock rem_lockspace -s $vgname:$host_id:/dev/$vgname/leases:0

	lvm lvchange -an $vgname/leases

	lvm lvremove $vgname/leases

	# vgremove
	lvm "$@"

	if [[ $? -ne 0 ]]; then
		exit 1
	fi
	
	exit 0

elif [[ "$cmd" == "vgremove" ]]; then

	get_args=$*
	get_vg_name

	check_vg_start

	if [[ "$glpath" == "" ]]; then
		echo "no global lock"
		lvm "$@"
	else
		export SANLVM_VGNAME=$vgname
		sanlock command -r sanlvm_global:lease:$glpath:1048576 -c ./sanlvm _vgremove "$@"
	fi

	if [[ $? -ne 0 ]]; then
		exit 1
	fi
	
	exit 0

elif [[ "$cmd" == "vgextend" ]]; then

	get_args=$*
	get_vg_name

	check_vg_start

	if [[ "$glpath" == "" ]]; then
		echo "no global lock"
		sanlock command -r $vgname:vglk:/dev/$vgname/leases:1048576 -c /sbin/lvm "$@"
	else
		sanlock command -r sanlvm_global:lease:$glpath:1048576 -r $vgname:vglk:/dev/$vgname/leases:1048576 -c /sbin/lvm "$@"
	fi

	if [[ $? -ne 0 ]]; then
		exit 1
	fi

	exit 0

elif [[ "$cmd" == "_lvcreate" ]]; then
	# wrapped by vglk
	shift

	get_args=$*
	get_vg_name
	get_lv_name

	offset=`sanlock direct next_free /dev/$vgname/leases`

	if [[ $? -ne 0 ]]; then
		echo "no space for new lease on /dev/$vgname/leases"
		exit 1
	fi

	# FIXME: if we use recent lvcreate arg to disable activation,
	# on creation, then we can remove the lvchange -an that follows.

	# lvcreate
	lvm "$@"

	if [[ $? -ne 0 ]]; then
		exit 1
	fi

	lvm lvchange -an $vgname/$lvname

	if [[ $? -ne 0 ]]; then
		exit 1
	fi

	lvm lvchange --addtag @sanlvm_offset_$offset $vgname/$lvname

	if [[ $? -ne 0 ]]; then
		exit 1
	fi

	sanlock direct init -r $vgname:$lvname:/dev/$vgname/leases:$offset

	if [[ $? -ne 0 ]]; then
		exit 1
	fi

	exit 0

elif [[ "$cmd" == "lvcreate" ]]; then

	get_args=$*
	get_vg_name

	check_vg_start

	if [[ "$no_active" == "--na" ]]; then
		echo "creating lv without activation lock"
		shift
		sanlock command -r $vgname:vglk:/dev/$vgname/leases:1048576 -c /sbin/lvm "$@"
	else
		sanlock command -r $vgname:vglk:/dev/$vgname/leases:1048576 -c ./sanlvm _lvcreate "$@"
	fi

	if [[ $? -ne 0 ]]; then
		exit 1
	fi

	exit 0

elif [[ "$cmd" == "_lvremove" ]]; then
	# wrapped by vglk
	shift

	vgname=$SANLVM_VGNAME
	lvname=$SANLVM_LVNAME
	offset=$SANLVM_OFFSET

	sanlvmd=`pidof sanlvmd`

	if [[ $? -ne 0 ]]; then
		echo "sanlvmd not found, required for lv locks"
		exit 1
	fi

	# refuse to remove the lv if the local host has the lv activated

	status=`sanlock client status | grep "r $vgname:$lvname"`

	if [[ "$status" != "" ]]; then
		echo "lv must be be deactivated before remove"
		exit 1
	fi

	# acquire ex activation lock on the lv so we know that no hosts have it active

	sanlock client acquire -r $vgname:$lvname:/dev/$vgname/leases:$offset -p $sanlvmd

	if [[ $? -ne 0 ]]; then
		echo "lv lock failed, may be active on another host"
		exit 1
	fi

	# lvremove
	lvm "$@"

	sanlock client release -r $vgname:$lvname:/dev/$vgname/leases:$offset -p $sanlvmd

	# clear the lease area that was used for the lv activation lease at offset
	dd if=/dev/zero of=/dev/$vgname/leases seek=`expr $offset / 512` bs=512 count=2048

	exit 0

elif [[ "$cmd" == "lvremove" ]]; then

	get_args=$*
	get_vg_lv_name
	get_lease_offset

	check_vg_start

	if [[ "$offset" == "" ]]; then
		echo "no lv activation lock for $vgname/$lvname, only protecting vg"
		sanlock command -r $vgname:vglk:/dev/$vgname/leases:1048576 -c /sbin/lvm "$@"
	else
		export SANLVM_VGNAME=$vgname
		export SANLVM_LVNAME=$lvname
		export SANLVM_OFFSET=$offset
		sanlock command -r $vgname:vglk:/dev/$vgname/leases:1048576 -c ./sanlvm _lvremove "$@"
	fi

	if [[ $? -ne 0 ]]; then
		exit 1
	fi

	exit 0

elif [[ "$cmd" == "_lvextend" ]] || [[ "$cmd" == "_lvreduce" ]] || [[ "$cmd" == "_lvconvert" ]] || [[ "$cmd" == "_lvresize" ]]; then
	# wrapped by vglk
	shift

	vgname=$SANLVM_VGNAME
	lvname=$SANLVM_LVNAME
	offset=$SANLVM_OFFSET

	sanlvmd=`pidof sanlvmd`

	if [[ $? -ne 0 ]]; then
		echo "sanlvmd not found, required for lv locks"
		exit 1
	fi

	# refuse to extend/reduce/convert the lv if the local host has the lv activate
	# FIXME: allow extend of active lv, not reduce

	status=`sanlock client status | grep "r $vgname:$lvname"`

	if [[ "$status" != "" ]]; then
		echo "lv must be be deactivated before extend/reduce/convert/resize"
		exit 1
	fi

	# acquire ex activation lock on the lv so we know that no hosts have it active

	sanlock client acquire -r $vgname:$lvname:/dev/$vgname/leases:$offset -p $sanlvmd

	if [[ $? -ne 0 ]]; then
		echo "lv lock failed, may be active on another host"
		exit 1
	fi

	# lvextend/lvreduce/lvconvert/lvresize
	lvm "$@"

	sanlock client release -r $vgname:$lvname:/dev/$vgname/leases:$offset -p $sanlvmd

	exit 0

elif [[ "$cmd" == "lvextend" ]] || [[ "$cmd" == "lvreduce" ]] || [[ "$cmd" == "lvconvert" ]] || [[ "$cmd" == "lvresize" ]]; then

	get_args=$*
	get_vg_lv_name
	get_lease_offset

	check_vg_start

	if [[ "$offset" == "" ]]; then
		echo "no lv activation lock for $vgname/$lvname, only protecting vg"
		sanlock command -r $vgname:vglk:/dev/$vgname/leases:1048576 -c /sbin/lvm "$@"
	else
		export SANLVM_VGNAME=$vgname
		export SANLVM_LVNAME=$lvname
		export SANLVM_OFFSET=$offset
		sanlock command -r $vgname:vglk:/dev/$vgname/leases:1048576 -c /sbin/lvm _$cmd "$@"
	fi

	if [[ $? -ne 0 ]]; then
		exit 1
	fi

	exit 0

elif [[ "$cmd" == "lvchange" ]]; then

	get_args=$*
	get_vg_lv_name
	get_active_mode
	get_lease_offset

	check_vg_start

	if [[ "$active_mode" != "y" ]] && [[ "$active_mode" != "n" ]]; then
		# non-activation related change to vg
		# are there other vg changes that would require vglk?
		echo "no locks needed for this command, use lvchange directly"
		exit 1
	fi

	if [[ "$active_mode" == "y" ]] && [[ "$lock_mode" == "" ]]; then
		echo "lock mode required, use sanlvm --ex|sh for vg/lv activation"
		exit 1
	fi

	if [[ "$active_mode" == "y" ]] && [[ "$lock_mode" != "--sh" ]] && [[ "$lock_mode" != "--ex" ]]; then
		echo "invalid lock mode, use sanlvm --ex|sh for vg/lv activation"
		exit 1
	fi

	if [[ "$active_mode" == "n" ]] && [[ "$lock_mode" != "" ]]; then
		echo "no lock mode required for vg/lv deactivation"
		exit 1
	fi

	if [[ "$lock_mode" == "--sh" ]]; then
		suffix=":SH"
	fi

	sanlvmd=`pidof sanlvmd`

	if [[ $? -ne 0 ]]; then
		echo "sanlvmd not found, required for lv locks"
		exit 1
	fi

	if [[ "$offset" == "" ]]; then
		echo "no lv activation lock for $vgname/$lvname, use lvchange directly"
		exit 1
	fi

	lvchange

	if [[ $? -ne 0 ]]; then
		exit 1
	fi

	exit 0

elif [[ "$cmd" == "vgchange" ]]; then

	get_args=$*
	get_vg_name
	get_active_mode

	check_vg_start

	if [[ "$active_mode" != "y" ]] && [[ "$active_mode" != "n" ]]; then
		# non-activation related change to vg
		# are there other vg changes that would require vglk?
		echo "no locks needed for this command, use vgchange directly"
		exit 1
	fi

	if [[ "$active_mode" == "y" ]] && [[ "$lock_mode" == "" ]]; then
		echo "lock mode required, use sanlvm --ex|sh for vg/lv activation"
		exit 1
	fi

	if [[ "$active_mode" == "y" ]] && [[ "$lock_mode" != "--sh" ]] && [[ "$lock_mode" != "--ex" ]]; then
		echo "invalid lock mode, use sanlvm --ex|sh for vg/lv activation"
		exit 1
	fi

	if [[ "$active_mode" == "n" ]] && [[ "$lock_mode" != "" ]]; then
		echo "no lock mode required for vg/lv deactivation"
		exit 1
	fi

	if [[ "$lock_mode" == "--sh" ]]; then
		suffix=":SH"
	fi

	sanlvmd=`pidof sanlvmd`

	if [[ $? -ne 0 ]]; then
		echo "sanlvmd not found, required for lv locks"
		exit 1
	fi

	for lvname in `lvs $vgname --noheadings -o lv_name`; do
		if [[ "$lvname" == "leases" ]]; then
			continue
		fi

		get_lease_offset

		if [[ "$offset" == "" ]]; then
			echo "no lv activation lock for $vgname/$lvname, use lvchange directly"
			continue
		fi

		lvchange
	done

	exit 0

elif [[ "$cmd" == "vgmerge" ]] || [[ "$cmd" == "vgsplit" ]] || [[ "$cmd" == "vgrename" ]] || [[ "$cmd" == "lvrename" ]]; then

	# FIXME: the vg commands could be implemented by combining
	# a global lease and vglk's

	echo "command not supported with sanlvm"
	exit 1

else
	if [[ $# -lt 2 ]]; then
		echo vg name required
		exit 1
	fi

	get_args=$*
	get_vg_name

	sanlock command -r $vgname:vglk:/dev/$vgname/leases:1048576 -c /sbin/lvm "$@"

	if [[ $? -ne 0 ]]; then
		exit 1
	fi

	exit 0
fi