path: root/config.d/10shareddisk.defconf

# Hey Emacs, this is a -*- shell-script -*- !!!

######################################################################

defconf SHAREDDISKSIZE "10G" \
	"<n>G" "size of the 3 shared disks"

defconf SHAREDDISK_COUNT "3" \
	"<n>" "the number of shared disks"

defconf SHAREDDISK_TEMPLATE "|shared_disk_template" \
	"<file>" "libvirt template fragment for shared disks"

defconf SHARED_DISK_TYPE "virtio" \
	"scsi|ide|virtio|iscsi" "type of disks to use for shared disks"

defconf SHARED_DISK_PREFIX "@uto" \
	"sd|hd|vd" "shared disk device prefix"

defconf SHARED_DISK_CACHE "default" \
	"default|none|writeback|writethrough" "shared disk cache type"

defconf SHARED_DISK_FIRST_SUFFIX "" \
	"|a|b|..." "override for 1st shared disk suffix - usually calculated by shared_disk_template()"

defconf SHARED_DISK_MULTIPATH_NUMPATHS "2" \
	"<n>" "number of paths to create for each shared disk"

defconf SHARED_DISK_MULTIPATH_CALLOUT "/sbin/scsi_id_autocluster.sh %n" \
	"<command>" ""

defconf SHARED_DISK_ID_GEN "shared_disk_id_default" \
	"<command>" "function/command for creating shared disk ids"

##########

defconf ISCSI_HOST "$KVMHOST" \
	"<ip-addr>" "host machine serving our iSCSI targets"

defconf ISCSI_PORT "3260" \
	"<port>" "port serving our iSCSI targets"

defconf ISCSI_IQN "iqn.1969-08" \
	"<string>" "iSCSI qualified name for targets"

defconf ISCSI_TID "1" \
	"<num>" "iSCSI target ID"

defconf ISCSI_DEFAULT_TEMPLATE "$installdir/templates/iSCSI-default" \
	"<file>" "template for iSCSI default file"

defconf ISCSI_ST_CONFIG_TEMPLATE "$installdir/templates/iSCSI-st_config" \
	"<file>" "template for iSCSI st_config file"


##############################

shared_disk_template ()
{
    # Don't do anything for iSCSI.
    [ "$SHARED_DISK_TYPE" != "iscsi" ] || return 0

    local paths="${1:-${SHARED_DISK_MULTIPATH_NUMPATHS}}"
    local first="${2:-${SHARED_DISK_FIRST_SUFFIX}}"

    if [ -z "$first" ] ; then
	if [ "$SYSTEM_DISK_PREFIX" = "$SHARED_DISK_PREFIX" ] ; then
	    first="b"
	else
	    first="a"
	fi
    fi
	
    local -a devices=($(eval echo {${first}..z} {a..z}{a..z}))

    local p SHARED_DISK_NUM
    local n=0
    for SHARED_DISK_NUM in $(seq 1 $SHAREDDISK_COUNT) ; do
	local DISK="${VIRTBASE}/${CLUSTER}/shared${SHARED_DISK_NUM}"
	run_hooks hack_disk_hooks "shared"
	local serial=$(awk "{ print \$${SHARED_DISK_NUM} }" <<<"$SHARED_DISK_IDS")
	for p in $(seq 1 $paths) ; do
	    local dev="${devices[$n]}"
	    [ -n "$dev" ] || die "Too many shared disks!  The function shared_disk_template needs to be hacked to allow more shared disks..."
	    cat <<EOF
    <disk type='file' device='disk'>
      <driver name='qemu' cache='@@SHARED_DISK_CACHE@@'/>
      <source file='${DISK}'/>
      <target dev='@@SHARED_DISK_PREFIX@@${dev}' bus='@@SHARED_DISK_TYPE@@'/>
      <serial>${serial}</serial>
      <shareable/>
    </disk>
EOF
	    n=$(($n + 1))
	done
    done
}

##############################

shared_disk_post_config_hook()
{
    if [ "$SHARED_DISK_PREFIX" = "@uto" ] ; then
	SHARED_DISK_PREFIX=$(rhel_disk_prefix $SHARED_DISK_TYPE)
    fi
}

register_hook post_config_hooks shared_disk_post_config_hook

register_hook create_cluster_hooks shared_disk_setup

shared_disk_setup ()
{
    SHARED_DISK_IDS=""
    _SHARED_DISK_TEMPLATE=""

    # Early exit if no shared disks
    have_shared_disks || return 0

    if [ "$SHARED_DISK_TYPE" = "iscsi" ] ; then
	iscsi_out="tmp/iscsi.${CLUSTER}"  # Global, not local.

	cat <<EOF >"$iscsi_out"
tgt-admin --delete tid=${ISCSI_TID} --force
tgtadm --lld iscsi --mode target --op new \
	--tid ${ISCSI_TID} -T ${ISCSI_IQN}.${CLUSTER}:tgtd
EOF
	local uc=$(echo "$CLUSTER" | tr a-z A-Z)
    fi

    echo "Creating ${SHAREDDISK_COUNT} shared disks"
    local SHARED_DISK_NUM
    for SHARED_DISK_NUM in $(seq 1 $SHAREDDISK_COUNT); do
	local DISK="$VIRTBASE/$CLUSTER/shared${SHARED_DISK_NUM}"
	run_hooks hack_disk_hooks "shared"
	local di="$DISK"
	if [ "$DISK_FOLLOW_SYMLINKS" = "yes" -a -L "$DISK" ] ; then
	    di=$(readlink "$DISK")
	fi
	rm -f "$di"
	local di_dirname="${di%/*}"
	mkdir -p "$di_dirname"

	if [ "$SHARED_DISK_TYPE" = "iscsi" ] ; then
	    dd if=/dev/zero seek=$SHAREDDISKSIZE bs=1 count=1 of="$di"

	    local paths=$SHARED_DISK_MULTIPATH_NUMPATHS  # readability
	    local zc=$(printf "%03d" $SHARED_DISK_NUM)
	    local p
	    for p in $(seq 1 $paths) ; do
		local lun=$(($paths * ($SHARED_DISK_NUM - 1) + $p))
		# vendor_id AUTCLSTR used by /etc/init.d/iscsimultipath
		cat <<EOF >>"$iscsi_out"
tgtadm --lld iscsi --mode logicalunit --op new \
    --tid ${ISCSI_TID} --lun ${lun} \
    -b ${DISK}
tgtadm --lld iscsi --mode logicalunit --op update \
    --tid ${ISCSI_TID} --lun ${lun} \
    --params vendor_id=AUTCLSTR,product_id=${uc},product_rev=0001,scsi_sn=SHARE${zc}
EOF
	    done
	else
	    qemu-img create -f raw "$di" $SHAREDDISKSIZE
	fi
	# setup a nice ID at the start of the disk
	"$SHARED_DISK_ID_GEN" "$SHARED_DISK_NUM" > tmp/diskid
	dd if=tmp/diskid of="$di" conv=notrunc bs=1 > /dev/null 2>&1
	local this_id=$(head -n 1 tmp/diskid)
	SHARED_DISK_IDS="${SHARED_DISK_IDS}${SHARED_DISK_IDS:+ }${this_id}"
    done
    echo

    if [ "$SHARED_DISK_TYPE" = "iscsi" ] ; then
	register_hook cluster_created_hooks iscsi_cluster_created
    fi
    register_hook setup_base_hooks shared_disk_setup_base
}

shared_disk_id_default ()
{
    # Disk number argument is ignored
    local t="AUTO-$(uuidgen)"
    # Length of 13 bytes is historical
    echo "${t:0:13}"
}

shared_disk_setup_base ()
{
    node_has_shared_disks "$type" || return 0

    if [ "$SHARED_DISK_TYPE" = "iscsi" ] ; then
	shared_disk_iscsi_setup_base
    else
	_SHARED_DISK_TEMPLATE="$SHAREDDISK_TEMPLATE"
    fi
}

##########

node_has_shared_disks_DEFAULT ()
{
    false
}

node_has_shared_disks ()
{
    local node_type="$1"

    call_func node_has_shared_disks "$node_type"
}

have_shared_disks ()
{
    [ -n "$SHAREDDISKSIZE" -a -n "$SHAREDDISK_TEMPLATE" -a \
	-n "$SHAREDDISK_COUNT" -a "$SHAREDDISK_COUNT" != 0 ] || \
	return 1

    local ret=false

    _check ()
    {
	if node_has_shared_disks "$1" ; then
	    ret=true
	fi
    }

    for_each_node _check

    $ret
}

is_dedicated_storage_node_DEFAULT ()
{
    false
}

have_dedicated_storage_nodes ()
{
    local ret=false

    _check ()
    {
	if call_func is_dedicated_storage_node "$1" ; then
	    ret=true
	fi
    }

    for_each_node _check

    $ret
}

##########

# Override this if you use a different scheme for IP addresses.
shared_disk_iscsi_setup_base_get_ip ()
{
    echo "${NETWORK_PRIVATE_PREFIX}.${IPNUM}"
}

shared_disk_iscsi_setup_base ()
{
    echo "Setting up iSCSI for shared disks"

    # This simulates what happens when you do something like:
    #   iscsiadm --mode discovery --type sendtargets --portal 10.0.0.1
    # provided only 1 target is accessible..

    local idir="/var/lib/iscsi"

    local dd="${idir}/nodes/${ISCSI_IQN}.${CLUSTER}:tgtd/${ISCSI_HOST},${ISCSI_PORT},1"
    local d="${dd}/default"
    diskimage mkdir_p "${dd}"
    diskimage substitute_vars "$ISCSI_DEFAULT_TEMPLATE" "$d"
    diskimage chmod 600 "$d"

    local sd="${idir}/send_targets/${ISCSI_HOST},${ISCSI_PORT}"
    local s="${sd}/st_config"
    diskimage mkdir_p "${sd}"
    diskimage substitute_vars "$ISCSI_ST_CONFIG_TEMPLATE" "$s"
    diskimage chmod 600 "$s"

    diskimage ln_s "$dd" "/${sd}/${ISCSI_IQN}.${ISCSI_IQN}.${CLUSTER}:tgtd,${ISCSI_HOST},${ISCSI_PORT},1,default"

    # After the iscsi initscript is run, new devices for multipath
    # don't seem to be noticed anymore.  This enables an initscript
    # that ensures that multipath is run when the desired number of
    # shared disk paths are available.
    diskimage command /sbin/chkconfig --add iscsimultipath

    local ip_addr=$(shared_disk_iscsi_setup_base_get_ip)
    cat <<EOF >>"$iscsi_out"
tgtadm --lld iscsi --op bind --mode target --tid ${ISCSI_TID} -I ${ip_addr}
EOF
}

iscsi_cluster_created ()
{
    cat <<EOF

This cluster uses iSCSI shared disks for the cluster file system.  The
commands to configure the iSCSI on the host machine are in the file
"${iscsi_out}".  You should now run this file on host $ISCSI_HOST
using "sh ${iscsi_out}".  The first command, which deletes an existing
iSCSI target, may fail if there is no existing iSCSI configuration.
However, all other commands should succeed.

You may also want to copy these commands somewhere so they are run
when $ISCSI_HOST boots (e.g. /etc/rc.local).
EOF
}