path: root/autocluster

#!/bin/bash
# main autocluster script
#
# Copyright (C) Andrew Tridgell  2008
# Copyright (C) Martin Schwenke  2008
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 3 of the License, or
# (at your option) any later version.
#   
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#   
# You should have received a copy of the GNU General Public License
# along with this program; if not, see <http://www.gnu.org/licenses/>.

##BEGIN-INSTALLDIR-MAGIC##
# There are better ways of doing this but not if you still want to be
# able to run straight out of a git tree.  :-)
if [ -f "$0" ]; then
    autocluster="$0"
else
    autocluster=$(which "$0")
fi
if [ -L "$autocluster" ] ; then
    autocluster=$(readlink "$autocluster")
fi
installdir=$(dirname "$autocluster")
##END-INSTALLDIR-MAGIC##

####################
# show program usage
usage ()
{
    cat <<EOF
Usage: autocluster [OPTION] ... <COMMAND>
  options:
     -c <file>                   specify config file (default is "config")
     -e <expr>                   execute <expr> and exit
     -E <expr>                   execute <expr> and continue
     -x                          enable script debugging
     --dump                      dump config settings and exit

  configuration options:
EOF

    usage_config_options

    cat <<EOF

  commands:
     base [ create | boot ] ...

     cluster [ build |
               destroy | undefine |
               create | update_hosts | boot | setup ] ...

     create base
           create a base image

     create cluster [ CLUSTERNAME ]
           create a full cluster

     create node CLUSTERNAME IP_OFFSET
           (re)create a single cluster node

     mount DISK
           mount a qemu disk on mnt/

     unmount | umount
           unmount a qemu disk from mnt/

     bootbase
           boot the base image
EOF
    exit 1
}

###############################

die () {
    if [ "$no_sanity" = 1 ] ; then
	fill_text 0 "WARNING: $*" >&2
    else
	fill_text 0 "ERROR: $*" >&2
	exit 1
    fi
}

announce ()
{
    echo "######################################################################"
    printf "# %-66s #\n" "$*"
    echo "######################################################################"
    echo ""
}

waitfor ()
{
    local file="$1"
    local msg="$2"
    local timeout="$3"

    local tmpfile=$(mktemp)

    cat <<EOF >"$tmpfile"
spawn tail -n 10000 -f $file
expect -timeout $timeout -re "$msg"
EOF

    export LANG=C
    expect "$tmpfile"
    rm -f "$tmpfile"

    if ! grep -E "$msg" "$file" > /dev/null; then
	echo "Failed to find \"$msg\" in \"$file\""
	return 1
    fi

    return 0
}

###############################

# Indirectly call a function named by ${1}_${2}
call_func () {
    local func="$1" ; shift
    local type="$1" ; shift

    local f="${func}_${type}"
    if type -t "$f" >/dev/null && ! type -P "$f" >/dev/null ; then
        "$f" "$@"
    else
        f="${func}_DEFAULT"
        if type -t "$f" >/dev/null && ! type -P "$f" >/dev/null  ; then
            "$f" "$type" "$@"
        else
	    die "No function defined for \"${func}\" \"${type}\""
        fi
    fi
}

# Note that this will work if you pass "call_func f" because the first
# element of the node tuple is the node type.  Nice...  :-)
for_each_node ()
{
    local n
    for n in $NODES ; do
	"$@" $(IFS=: ; echo $n)
    done
}

node_is_ctdb_node_DEFAULT ()
{
    echo 0
}

hack_one_node_with ()
{
    local filter="$1" ; shift

    local node_type="$1"
    local ip_offset="$2"
    local name="$3"
    local ctdb_node="$4"

    $filter

    local item="${node_type}:${ip_offset}${name:+:}${name}${ctdb_node:+:}${ctdb_node}"
    nodes="${nodes}${nodes:+ }${item}"
}

# This also gets used for non-filtering iteration.
hack_all_nodes_with ()
{
    local filter="$1"

    local nodes=""
    for_each_node hack_one_node_with "$filter"
    NODES="$nodes"
}

register_hook ()
{
    local hook_var="$1"
    local new_hook="$2"

    eval "$hook_var=\"${!hook_var}${!hook_var:+ }${new_hook}\""
}

run_hooks ()
{
    local hook_var="$1"
    shift

    local i
    for i in ${!hook_var} ; do
	$i "$@"
    done
}

# Use with care, since this may clear some autocluster defaults.!
clear_hooks ()
{
    local hook_var="$1"

    eval "$hook_var=\"\""
}

##############################

# These hooks are intended to customise the value of $DISK.  They have
# access to 1 argument ("base", "system", "shared") and the variables
# $VIRTBASE, $CLUSTER, $BASENAME (for "base"), $NAME (for "system"),
# $SHARED_DISK_NUM (for "shared").  A hook must be deterministic and
# should not be stateful, since they can be called multiple times for
# the same disk.
hack_disk_hooks=""

create_node_DEFAULT ()
{
    local type="$1"
    local ip_offset="$2"
    local name="$3"
    local ctdb_node="$4"

    echo "Creating node \"$name\" (of type \"${type}\")"

    create_node_COMMON "$name" "$ip_offset" "$type"
}

# common node creation stuff
create_node_COMMON ()
{
    local NAME="$1"
    local ip_offset="$2"
    local type="$3"
    local template_file="${4:-$NODE_TEMPLATE}"

    if [ "$SYSTEM_DISK_FORMAT" != "qcow2" -a "$BASE_FORMAT" = "qcow2" ] ; then
	die "Error: if BASE_FORMAT is \"qcow2\" then SYSTEM_DISK_FORMAT must also be \"qcow2\"."
    fi

    local IPNUM=$(($FIRSTIP + $ip_offset))
    make_network_map

    # Determine base image name.  We use $DISK temporarily to allow
    # the path to be hacked.
    local DISK="${VIRTBASE}/${BASENAME}.${BASE_FORMAT}"
    if [ "$BASE_PER_NODE_TYPE" = "yes" ] ; then
	DISK="${VIRTBASE}/${BASENAME}-${type}.${BASE_FORMAT}"
    fi
    run_hooks hack_disk_hooks "base"
    local base_disk="$DISK"

    # Determine the system disk image name.
    DISK="${VIRTBASE}/${CLUSTER}/${NAME}.${SYSTEM_DISK_FORMAT}"
    run_hooks hack_disk_hooks "system"

    local di="$DISK"
    if [ "$DISK_FOLLOW_SYMLINKS" = "yes" -a -L "$DISK" ] ; then
	di=$(readlink "$DISK")
    fi
    rm -f "$di"
    local di_dirname="${di%/*}"
    mkdir -p "$di_dirname"

    case "$SYSTEM_DISK_FORMAT" in
	qcow2)
	    echo "Creating the disk..."
	    qemu-img create -b "$base_disk" -f qcow2 "$di"
	    create_node_configure_image "$DISK" "$type"
	    ;;
	raw)
	    echo "Creating the disk..."
	    cp -v --sparse=always "$base_disk" "$di"
	    create_node_configure_image "$DISK" "$type"
	    ;;
	reflink)
	    echo "Creating the disk..."
	    cp -v --reflink=always "$base_disk" "$di"
	    create_node_configure_image "$DISK" "$type"
	    ;;
	mmclone)
	    echo "Creating the disk (using mmclone)..."
	    local base_snap="${base_disk}.snap"
	    [ -f "$base_snap" ] || mmclone snap "$base_disk" "$base_snap"
	    mmclone copy "$base_snap" "$di"
	    create_node_configure_image "$DISK" "$type"
	    ;;
	none)
	    echo "Skipping disk image creation as requested"
	    ;;
	*)
	    die "Error: unknown SYSTEM_DISK_FORMAT=\"${SYSTEM_DISK_FORMAT}\"."
    esac

    # Pull the UUID for this node out of the map.
    UUID=$(awk "\$1 == $ip_offset {print \$2}" $uuid_map)
    
    mkdir -p tmp

    echo "Creating $NAME.xml"
    substitute_vars $template_file tmp/$NAME.xml
    
    # install the XML file
    $VIRSH undefine $NAME > /dev/null 2>&1 || true
    $VIRSH define tmp/$NAME.xml
}

create_node_configure_image ()
{
    local disk="$1"
    local type="$2"

    diskimage mount "$disk"
    setup_base "$type"
    diskimage unmount
}

hack_network_map_hooks=""

# Uses: CLUSTER, NAME, NETWORKS, FIRSTIP, ip_offset
make_network_map ()
{
    network_map="tmp/network_map.$NAME"

    if [ -n "$CLUSTER" ] ; then
	local md5=$(echo "$CLUSTER" | md5sum)
	local nh=$(printf "%02x" $ip_offset)
	local mac_prefix="02:${md5:0:2}:${md5:2:2}:00:${nh}:"
    else
	local mac_prefix="02:42:42:00:00:"
    fi

    local n
    local count=1
    for n in $NETWORKS ; do
	local ch=$(printf "%02x" $count)
	local mac="${mac_prefix}${ch}"

	set -- ${n//,/ }
	local ip_bits="$1" ; shift
	local dev="$1" ; shift
	local opts="$*"

	local net="${ip_bits%/*}"
	local netname="acnet_${net//./_}"

	local ip="${net%.*}.${IPNUM}"
	local mask="255.255.255.0"

	# This can be used to override the variables in the echo
	# statement below.  The hook can use any other variables
	# available in this function.
	run_hooks hack_network_map_hooks

	echo "${netname} ${dev} ${ip} ${mask} ${mac} ${opts}"
	count=$(($count + 1))
    done >"$network_map"
}

##############################

expand_nodes ()
{
    # Expand out any abbreviations in NODES.
    local ns=""
    local n
    for n in $NODES ; do
	local t="${n%:*}"
	local ips="${n#*:}"
	case "$ips" in
	    *,*)
		local i
		for i in ${ips//,/ } ; do
		    ns="${ns}${ns:+ }${t}:${i}"
		done
		;;
	    *-*)
		local i
		for i in $(seq ${ips/-/ }) ; do
		    ns="${ns}${ns:+ }${t}:${i}"
		done
		;;
	    *)
		ns="${ns}${ns:+ }${n}"
	esac
    done
    NODES="$ns"

    # Check IP addresses for duplicates.
    local ip_offsets=":"
    # This function doesn't modify anything...
    get_ip_offset ()
    {
	[ "${ip_offsets/${ip_offset}}" != "$ip_offsets" ] && \
	    die "Duplicate IP offset in NODES - ${node_type}:${ip_offset}"
	ip_offsets="${ip_offsets}${ip_offset}:"
    }
    hack_all_nodes_with get_ip_offset

    # Determine node names and whether they're in the CTDB cluster
    declare -A node_count
    _get_name_ctdb_node ()
    {
	local count=$((${node_count[$node_type]:-0} + 1))
	node_count[$node_type]=$count
	name=$(call_func node_name_format "$node_type" "$CLUSTER" $count) || {
	    echo "ERROR: Node type \"${node_type}\" not defined!"
	    echo "Valid node types are:"
	    set | sed -n 's@^node_name_format_\(.*\) ().*@  \1@p'
	    exit 1
	}
	ctdb_node=$(call_func node_is_ctdb_node "$node_type")
    }
    hack_all_nodes_with _get_name_ctdb_node
}

##############################

sanity_check_cluster_name ()
{
    [ -z "${CLUSTER//[A-Za-z0-9]}" ] || \
	die "Cluster names should be restricted to the characters A-Za-z0-9.  \
Some cluster filesystems have problems with other characters."
}

hosts_file=

cluster_nodelist_hacking ()
{
    # Rework the NODES list
    expand_nodes

    # Build /etc/hosts and hack the names of the ctdb nodes
    hosts_line_hack_name ()
    {
	local sname=""
	local hosts_line
	local ip_addr="${NETWORK_PRIVATE_PREFIX}.$(($FIRSTIP + $ip_offset))"

	# Primary name for CTDB nodes is <CLUSTER>n<num>
	if [ "$ctdb_node" = 1 ] ; then
	    num_ctdb_nodes=$(($num_ctdb_nodes + 1))
	    sname="${CLUSTER}n${num_ctdb_nodes}"
	    hosts_line="$ip_addr ${sname}.${ld} ${name}.${ld} $name $sname"
	    name="$sname"
	else
	    hosts_line="$ip_addr ${name}.${ld} $name"
	fi

	# This allows you to add a function to your configuration file
	# to modify hostnames (and other aspects of nodes).  This
	# function can access/modify $name (the existing name),
	# $node_type and $ctdb_node (1, if the node is a member of the
	# CTDB cluster, 0 otherwise).
	if [ -n "$HOSTNAME_HACKING_FUNCTION" ] ; then
	    local old_name="$name"
	    $HOSTNAME_HACKING_FUNCTION
	    if [ "$name" != "$old_name" ] ; then
		hosts_line="$ip_addr ${name}.${ld} $name"
	    fi
	fi

	echo "$hosts_line"
    }
    hosts_file="tmp/hosts.$CLUSTER"
    {
	local num_ctdb_nodes=0
	local ld=$(echo $DOMAIN | tr A-Z a-z)
	echo "# autocluster $CLUSTER"
	hack_all_nodes_with hosts_line_hack_name
	echo
    } >$hosts_file

    # Build /etc/ctdb/nodes
    ctdb_nodes_line ()
    {
	[ "$ctdb_node" = 1 ] || return 0
	echo "${NETWORK_PRIVATE_PREFIX}.$(($FIRSTIP + $ip_offset))"
	num_nodes=$(($num_nodes + 1))
    }
    nodes_file="tmp/nodes.$CLUSTER"
    local num_nodes=0
    hack_all_nodes_with ctdb_nodes_line >$nodes_file

    # Build UUID map
    uuid_map="tmp/uuid_map.$CLUSTER"
    uuid_map_line ()
    {
	echo "${ip_offset} $(uuidgen) ${node_type}"
    }
    hack_all_nodes_with uuid_map_line >$uuid_map
}

create_cluster_hooks=
cluster_created_hooks=

cluster_create ()
{
    # Use $1.  If not set then use value from configuration file.
    CLUSTER="${1:-${CLUSTER}}"
    announce "cluster create \"${CLUSTER}\""
    [ -n "$CLUSTER" ] || die "\$CLUSTER not set"

    sanity_check_cluster_name

    mkdir -p $VIRTBASE/$CLUSTER $KVMLOG tmp

    # Run hooks before doing anything else.
    run_hooks create_cluster_hooks

    for_each_node call_func create_node

    echo "Cluster $CLUSTER created"
    echo ""

    run_hooks cluster_created_hooks
}

cluster_created_hosts_message ()
{
    echo "You may want to add this to your /etc/hosts file:"
    cat $hosts_file
}

register_hook cluster_created_hooks cluster_created_hosts_message

cluster_destroy ()
{
    announce "cluster destroy \"${CLUSTER}\""
    [ -n "$CLUSTER" ] || die "\$CLUSTER not set"

    vircmd destroy "$CLUSTER" || true
}

cluster_undefine ()
{
    announce "cluster undefine \"${CLUSTER}\""
    [ -n "$CLUSTER" ] || die "\$CLUSTER not set"

    vircmd undefine "$CLUSTER" || true
}

cluster_update_hosts ()
{
    announce "cluster update_hosts \"${CLUSTER}\""
    [ -n "$CLUSTER" ] || die "\$CLUSTER not set"

    [ -n "$hosts_file" ] || hosts_file="tmp/hosts.${CLUSTER}"
    [ -r "$hosts_file" ] || die "Missing hosts file \"${hosts_file}\""

    # Building a general node name regexp is a bit cumbersome.  :-)
    local name_regexp="("
    for i in $(set | sed -n -e "s@^\(node_name_format_.*\) ().*@\1@p") ; do
	# Format node name with placeholders (remembering that "_" is
	# not valid in a real cluster name)
	local t=$("$i" "_" "0")
	# now replace the placeholders with regexps - order is
	# important here, since the cluster name can contain digits
	t=$(sed -r -e "s@[[:digit:]]+@[[:digit:]]+@" -e "s@_@${CLUSTER}@" <<<"$t")
	# add to the regexp
	name_regexp="${name_regexp}${t}|"
    done
    name_regexp="${name_regexp}${CLUSTER}n[[:digit:]]+)"

    local pat="# autocluster ${CLUSTER}\$|[[:space:]]${name_regexp}"

    local t="/etc/hosts.${CLUSTER}"
    grep -E "$pat" /etc/hosts >"$t" || true
    if diff -B "$t" "$hosts_file" >/dev/null ; then
	rm "$t"
	return
    fi

    local old=/etc/hosts.old.autocluster
    cp /etc/hosts "$old"
    local new=/etc/hosts.new
    grep -Ev "$pat" "$old" |
    cat -s - "$hosts_file" >"$new"

    mv "$new" /etc/hosts

    echo "Made these changes to /etc/hosts:"
    diff -u "$old" /etc/hosts || true
}

cluster_boot ()
{
    [ -n "$CLUSTER_PATTERN" ] || CLUSTER_PATTERN="$CLUSTER"
    announce "cluster boot \"${CLUSTER_PATTERN}\""
    [ -n "$CLUSTER_PATTERN" ] || die "\$CLUSTER_PATTERN not set"

    vircmd start "$CLUSTER_PATTERN"

    local nodes=$(vircmd dominfo "$CLUSTER_PATTERN" 2>/dev/null | \
	sed -n -e 's/Name: *//p')

    # Wait for each node
    local i
    for i in $nodes ; do
	waitfor "${KVMLOG}/serial.$i" "login:" 300 || {
	    vircmd destroy "$CLUSTER_PATTERN"
	    die "Failed to create cluster"
	}
    done

    # Move past the last line of log output
    echo ""
}

cluster_setup_tasks_DEFAULT ()
{
    local stage="$1"

    # By default nodes have no tasks
    case "$stage" in
	install_packages) echo "" ;;
	setup_clusterfs)  echo "" ;;
	setup_node)       echo "" ;;
	setup_cluster)    echo "" ;;
    esac
}

cluster_setup ()
{
    announce "cluster setup \"${CLUSTER}\""
    [ -n "$CLUSTER" ] || die "\$CLUSTER not set"

    local ssh="ssh -o StrictHostKeyChecking=no"
    local setup_clusterfs_done=false
    local setup_cluster_done=false

    _cluster_setup_do_stage ()
    {
	local stage="$1"
	local type="$2"
	local ip_offset="$3"
	local name="$4"
	local ctdb_node="$5"

	local tasks=$(call_func cluster_setup_tasks "$type" "$stage")

	if [ -n "$tasks" ] ; then
	    # These tasks are only done on 1 node
	    case "$stage" in
		setup_clusterfs)
		    if $setup_clusterfs_done ; then
			return
		    else
			setup_clusterfs_done=true
		    fi
		    ;;
		setup_cluster)
		    if $setup_cluster_done ; then
			return
		    else
			setup_cluster_done=true
		    fi
		    ;;
	    esac

	    $ssh "$name" ./scripts/cluster_setup.sh "$stage" $tasks
	fi

    }

    local stages="install_packages setup_clusterfs setup_node setup_cluster"
    local stage
    for stage in $stages ; do
	for_each_node _cluster_setup_do_stage "$stage"
    done
}

create_one_node ()
{
    CLUSTER="$1"
    local single_node_ip_offset="$2"

    sanity_check_cluster_name

    mkdir -p $VIRTBASE/$CLUSTER $KVMLOG tmp

    for n in $NODES ; do
	set -- $(IFS=: ; echo $n)
	[ $single_node_ip_offset -eq $2 ] || continue
	call_func create_node "$@"
	
	echo "Requested node created"
	echo ""
	echo "You may want to update your /etc/hosts file:"
	cat $hosts_file
	
	break
    done
}

###############################
# test the proxy setup
test_proxy() {
    export http_proxy=$WEBPROXY
    wget -O /dev/null $INSTALL_SERVER || \
        die "Your WEBPROXY setting \"$WEBPROXY\" is not working"
    echo "Proxy OK"
}

###################

kickstart_floppy_create_hooks=

guess_install_network ()
{
    # Figure out IP address to use during base install.  Default to
    # the IP address of the 1st (private) network. If a gateway is
    # specified then use the IP address associated with it.
    INSTALL_IP=""
    INSTALL_GW=""
    local netname dev ip mask mac opts
    while read netname dev ip mask mac opts; do
	local o
	for o in $opts ; do
	    case "$o" in
		gw\=*)
		    INSTALL_GW="${o#gw=}"
		    INSTALL_IP="$ip"
	    esac
	done
	[ -n "$INSTALL_IP" ] || INSTALL_IP="$ip"
    done <"$network_map"
}

# create base image
base_create()
{
    local NAME="$BASENAME"
    local DISK="${VIRTBASE}/${NAME}.${BASE_FORMAT}"
    run_hooks hack_disk_hooks "base"

    mkdir -p $KVMLOG

    echo "Testing WEBPROXY $WEBPROXY"
    test_proxy

    local di="$DISK"
    if [ "$DISK_FOLLOW_SYMLINKS" = "yes" -a -L "$DISK" ] ; then
	di=$(readlink "$DISK")
    fi
    rm -f "$di"
    local di_dirname="${di%/*}"
    mkdir -p "$di_dirname"

    echo "Creating the disk"
    qemu-img create -f $BASE_FORMAT "$di" $DISKSIZE

    rm -rf tmp
    mkdir -p mnt tmp tmp/ISO

    setup_timezone

    IPNUM=$FIRSTIP
    make_network_map

    guess_install_network

    echo "Creating kickstart file from template"
    substitute_vars "$KICKSTART" "tmp/ks.cfg"

    # $ISO gets $ISO_DIR prepended if it doesn't start with a leading '/'.
    case "$ISO" in
	(/*) : ;;
	(*) ISO="${ISO_DIR}/${ISO}"
    esac
    
    echo "Creating kickstart floppy"
    dd if=/dev/zero of=tmp/floppy.img bs=1024 count=1440
    mkdosfs -n KICKSTART tmp/floppy.img
    mount -o loop -t msdos tmp/floppy.img mnt
    cp tmp/ks.cfg mnt
    mount -o loop,ro $ISO tmp/ISO
    
    echo "Setting up bootloader"
    cp tmp/ISO/isolinux/isolinux.bin tmp
    cp tmp/ISO/isolinux/vmlinuz tmp
    cp tmp/ISO/isolinux/initrd.img tmp

    run_hooks kickstart_floppy_create_hooks

    umount tmp/ISO
    umount mnt

    UUID=`uuidgen`

    substitute_vars $INSTALL_TEMPLATE tmp/$NAME.xml

    rm -f $KVMLOG/serial.$NAME

    # boot the install CD
    $VIRSH create tmp/$NAME.xml

    echo "Waiting for install to start"
    sleep 2
    
    # wait for the install to finish
    if ! waitfor $KVMLOG/serial.$NAME "$KS_DONE_MESSAGE" $CREATE_BASE_TIMEOUT ; then
	$VIRSH destroy $NAME
	die "Failed to create base image ${DISK} after waiting for ${CREATE_BASE_TIMEOUT} seconds.
You may need to increase the value of CREATE_BASE_TIMEOUT.
Alternatively, the install might have completed but KS_DONE_MESSAGE
(currently \"${KS_DONE_MESSAGE}\")
may not have matched anything at the end of the kickstart output."
    fi
    
    $VIRSH destroy $NAME

    ls -l $DISK
    cat <<EOF

Install finished, base image $DISK created

You may wish to run
   chcon -t virt_content_t $DISK
   chattr +i $DISK
To ensure that this image does not change

Note that the root password has been set to $ROOTPASSWORD

EOF
}

###############################
# boot the base disk
base_boot() {
    rm -rf tmp
    mkdir -p tmp

    NAME="$BASENAME"
    DISK="${VIRTBASE}/${NAME}.${BASE_FORMAT}"

    IPNUM=$FIRSTIP

    make_network_map

    CLUSTER="base"

    diskimage mount $DISK
    setup_base
    diskimage unmount

    UUID=`uuidgen`
    
    echo "Creating $NAME.xml"
    substitute_vars $BOOT_TEMPLATE tmp/$NAME.xml
    
    # boot the base system
    $VIRSH create tmp/$NAME.xml
}

######################################################################

# Updating a disk image...

diskimage ()
{
    local func="$1"
    shift
    call_func diskimage_"$func" "$SYSTEM_DISK_ACCESS_METHOD" "$@"
}

# setup the files from $BASE_TEMPLATES/, substituting any variables
# based on the config
copy_base_dir_substitute_templates ()
{
    local dir="$1"

    local d="$BASE_TEMPLATES/$dir"
    [ -d "$d" ] || return 0

    local f
    for f in $(cd "$d" && find . \! -name '*~' \( -type d -name .svn -prune -o -print \) ) ; do
	f="${f#./}" # remove leading "./" for clarity
	if [ -d "$d/$f" ]; then
	    # Don't chmod existing directory
	    if diskimage is_directory "/$f" ; then
		continue
	    fi
	    diskimage mkdir_p "/$f"
	else
	    echo " Install: $f"
	    diskimage substitute_vars "$d/$f" "/$f"
	fi
	diskimage chmod_reference "$d/$f" "/$f"
    done
}

setup_base_hooks=

setup_base_ssh_keys ()
{
    # this is needed as git doesn't store file permissions other
    # than execute
    # Note that we protect the wildcards from the local shell.
    diskimage chmod 600 "/etc/ssh/*key" "/root/.ssh/*"
    diskimage chmod 700 "/etc/ssh" "/root/.ssh" "/root"
    if [ -r "$HOME/.ssh/id_rsa.pub" ]; then
       echo "Adding $HOME/.ssh/id_rsa.pub to ssh authorized_keys"
       diskimage append_text_file "$HOME/.ssh/id_rsa.pub" "/root/.ssh/authorized_keys"
    fi
    if [ -r "$HOME/.ssh/id_dsa.pub" ]; then
       echo "Adding $HOME/.ssh/id_dsa.pub to ssh authorized_keys"
       diskimage append_text_file "$HOME/.ssh/id_dsa.pub" "/root/.ssh/authorized_keys"
    fi
}

register_hook setup_base_hooks setup_base_ssh_keys

setup_base_grub_conf ()
{
    echo "Adjusting grub.conf"
    local o="$EXTRA_KERNEL_OPTIONS" # For readability.
    local grub_configs="/boot/grub/grub.conf"
    if ! diskimage is_file "$grub_configs" ; then
	grub_configs="/etc/default/grub /boot/grub2/grub.cfg"
    fi
    local c
    for c in $grub_configs ; do
	diskimage sed "$c" \
	    -e "s/console=ttyS0,19200/console=ttyS0,115200/"  \
	    -e "s/ console=tty1//" -e "s/ rhgb/ norhgb/"  \
	    -e "s/ nodmraid//" -e "s/ nompath//"  \
	    -e "s/quiet/noapic divider=10${o:+ }${o}/g"
    done
}

register_hook setup_base_hooks setup_base_grub_conf

setup_base()
{
    local type="$1"

    umask 022
    echo "Copy base files"
    copy_base_dir_substitute_templates "all"
    if [ -n "$type" ] ; then
	copy_base_dir_substitute_templates "$type"
    fi

    run_hooks setup_base_hooks
}

ipv4_prefix_to_netmask ()
{
    local prefix="$1"

    local div=$(($prefix / 8))
    local mod=$(($prefix % 8))

    local octet
    for octet in 1 2 3 4 ; do
	if [ $octet -le $div ] ; then
	    echo -n "255"
	elif [ $mod -ne 0 -a $octet -eq $(($div + 1)) ] ; then
	    local shift=$((8 - $mod))
	    echo -n $(( (255 >> $shift << $shift) ))
	else
	    echo -n 0
	fi
	if [ $octet -lt 4 ] ; then
	    echo -n '.'
	fi
    done

    echo
}

# setup various networking components
setup_network()
{
    # This avoids doing anything when we're called from boot_base().
    if [ -z "$hosts_file" ] ; then
	echo "Skipping network-related setup"
	return
    fi

    echo "Setting up networks"
    diskimage append_text_file "$hosts_file" "/etc/hosts"

    echo "Setting up /etc/ctdb/nodes"
    diskimage mkdir_p "/etc/ctdb"
    diskimage put "$nodes_file" "/etc/ctdb/nodes"

    [ "$WEBPROXY" = "" ] || {
	diskimage append_text "export http_proxy=$WEBPROXY" "/etc/bashrc"
    }

    if [ -n "$NFSSHARE" -a -n "$NFS_MOUNTPOINT" ] ; then
	echo "Enabling nfs mount of $NFSSHARE"
	diskimage mkdir_p "$NFS_MOUNTPOINT"
	diskimage append_text "$NFSSHARE $NFS_MOUNTPOINT nfs nfsvers=3,intr 0 0" "/etc/fstab"
    fi

    diskimage mkdir_p "/etc/yum.repos.d"
    echo '@@@YUM_TEMPLATE@@@' | diskimage substitute_vars - "/etc/yum.repos.d/autocluster.repo"

    diskimage rm_rf "/etc/udev/rules.d/70-persistent-net.rules"

    echo "Setting up network interfaces: "
    local netname dev ip mask mac opts
    while read netname dev ip mask mac opts; do
	echo "  $dev"

	local o gw
	gw=""
	for o in $opts ; do
	    case "$o" in
		gw\=*)
		    gw="${o#gw=}"
	    esac
	done

	cat <<EOF | \
	    diskimage put - "/etc/sysconfig/network-scripts/ifcfg-${dev}"
DEVICE=$dev
ONBOOT=yes
TYPE=Ethernet
IPADDR=$ip
NETMASK=$mask
HWADDR=$mac
${gw:+GATEWAY=}${gw}
EOF

	# This goes to 70-persistent-net.rules
	cat <<EOF
# Generated by autocluster
SUBSYSTEM=="net", ACTION=="add", DRIVERS=="?*", ATTR{address}=="${mac}", ATTR{type}=="1", KERNEL=="eth*", NAME="${dev}"

EOF
    done <"$network_map" |
    diskimage put - "/etc/udev/rules.d/70-persistent-net.rules"
}

register_hook setup_base_hooks setup_network

setup_base_cluster_setup_config ()
{
    local f
    {
	echo "# Generated by autocluster"
	echo
	# This is a bit of a hack.  Perhaps these script belong
	# elsewhere, since they no longer have templates?
	for f in $(find "${BASE_TEMPLATES}/all/root/scripts" -type f |
	    xargs grep -l '^#config:') ; do

	    b=$(basename "$f")
	    echo "# $b"
	    local vs v
	    vs=$(sed -n 's@^#config: *@@p' "$f")
	    for v in $vs ; do
		# This could substitute the values in directly using
		# ${!v} but then no sanity checking is done to make
		# sure variables are set.
		echo "${v}=\"@@${v}@@\""
	    done
	    echo
	done
    } | diskimage substitute_vars - "/root/scripts/cluster_setup.config"
}

register_hook setup_base_hooks setup_base_cluster_setup_config

setup_timezone() {
    [ -z "$TIMEZONE" ] && {
	[ -r /etc/timezone ] && {
	    TIMEZONE=`cat /etc/timezone`
	}
	[ -r /etc/sysconfig/clock ] && {
	    . /etc/sysconfig/clock
	    TIMEZONE="$ZONE"
	}
	TIMEZONE="${TIMEZONE// /_}"
    }
    [ -n "$TIMEZONE" ] || \
	die "Unable to determine TIMEZONE - please set in config"
}

# substite a set of variables of the form @@XX@@ for the shell
# variables $XX in a file.
#
# Indirect variables @@@XX@@@ (3 ats) specify that the variable should
# contain a filename whose contents are substituted, with variable
# substitution applied to those contents.  If filename starts with '|'
# it is a command instead - however, quoting is extremely fragile.
substitute_vars() {(
	infile="${1:-/dev/null}" # if empty then default to /dev/null
	outfile="$2" # optional

	tmp_out=$(mktemp)
	cat "$infile" >"$tmp_out"

	# Handle any indirects by looping until nothing changes.
	# However, only handle 10 levels of recursion.
	count=0
	while : ; do
	    if ! _substitute_vars "$tmp_out" "@@@" ; then
		rm -f "$tmp_out"
		die "Failed to expand template $infile"
	    fi

	    # No old version of file means no changes made.
	    if [ ! -f "${tmp_out}.old" ] ; then
		break
	    fi

	    rm -f "${tmp_out}.old"

	    count=$(($count + 1))
	    if [ $count -ge 10 ] ; then
		rm -f "$tmp_out"
		die "Recursion too deep in $infile - only 10 levels allowed!"
	    fi
	done

	# Now regular variables.
	if ! _substitute_vars "$tmp_out" "@@" ; then
	    rm -f "$tmp_out"
	    die "Failed to expand template $infile"
	fi
	rm -f "${tmp_out}.old"

	if [ -n "$outfile" ] ; then
	    mv "$tmp_out" "$outfile"
	else
	    cat "$tmp_out"
	    rm -f "$tmp_out"
	fi
)}


# Delimiter @@ means to substitute contents of variable.
# Delimiter @@@ means to substitute contents of file named by variable.
# @@@ supports leading '|' in variable value, which means to excute a
# command.
_substitute_vars() {(
	tmp_out="$1"
	delimiter="${2:-@@}"

	# Get the list of variables used in the template.  The grep
	# gets rid of any blank lines and lines with extraneous '@'s
	# next to template substitutions.
	VARS=$(sed -n -e "s#[^@]*${delimiter}\([A-Z0-9_][A-Z0-9_]*\)${delimiter}[^@]*#\1\n#gp" "$tmp_out" |
	    grep '^[A-Z0-9_][A-Z0-9_]*$' |
	    sort -u)

	tmp=$(mktemp)
	for v in $VARS; do
	    # variable variables are fun .....
	    [ "${!v+x}" ] || {
		rm -f $tmp
		die "No substitution given for ${delimiter}$v${delimiter} in $infile"
	    }
	    s=${!v}

	    if [ "$delimiter" = "@@@" ] ; then
		f=${s:-/dev/null}
		c="${f#|}" # Is is a command, signified by a leading '|'?
		if [ "$c" = "$f" ] ; then
		    # No leading '|', cat file.
		    s=$(cat -- "$f")
		    [ $? -eq 0 ] || {
			rm -f $tmp
			die "Could not substitute contents of file $f"
		    }
		else
		    # Leading '|', execute command.
		    # Quoting problems here - using eval "$c" doesn't help.
		    s=$($c)
		    [ $? -eq 0 ] || {
			rm -f $tmp
			die "Could not execute command $c"
		    }
		fi
	    fi

	    # escape some pesky chars
	    # This first one can be too slow if done using a bash
	    # variable pattern subsitution.
	    s=$(echo -n "$s" | tr '\n' '\001' | sed -e 's/\o001/\\n/g')
	    s=${s//#/\\#}
	    s=${s//&/\\&}
	    echo "s#${delimiter}${v}${delimiter}#${s}#g"
	done > $tmp

	# Get the in-place sed to make a backup of the old file.
	# Remove the backup if it is the same as the resulting file -
	# this acts as a flag to the caller that no changes were made.
	sed -i.old -f $tmp "$tmp_out"
	if cmp -s "${tmp_out}.old" "$tmp_out" ; then
	    rm -f "${tmp_out}.old"
	fi

	rm -f $tmp
)}

check_command() {
    which $1 > /dev/null || die "Please install $1 to continue"
}

# Set a variable if it isn't already set.  This allows environment
# variables to override default config settings.
defconf() {
    local v="$1"
    local e="$2"

    [ "${!v+x}" ] || eval "$v=\"$e\""
}

load_config () {
    local i

    for i in "${installdir}/config.d/"*.defconf ; do
	. "$i"
    done
}

# Print the list of config variables defined in config.d/.
get_config_options () {( # sub-shell for local declaration of defconf()
	local options=
	defconf() { options="$options $1" ; }
	load_config
	echo $options
)}

# Produce a list of long options, suitable for use with getopt, that
# represent the config variables defined in config.d/.
getopt_config_options () {
    local x=$(get_config_options | tr 'A-Z_' 'a-z-')
    echo "${x// /:,}:"
}

# Unconditionally set the config variable associated with the given
# long option.
setconf_longopt () {
    local longopt="$1"
    local e="$2"

    local v=$(echo "${longopt#--}" | tr 'a-z-' 'A-Z_')
    # unset so defconf will set it
    eval "unset $v"
    defconf "$v" "$e"
}

# Dump all of the current config variables.
dump_config() {
    local o
    for o in $(get_config_options) ; do
	echo "${o}=\"${!o}\""
    done
    exit 0
}

# $COLUMNS is set in interactive bash shells.  It probably isn't set
# in this shell, so let's set it if it isn't.
: ${COLUMNS:=$(stty size 2>/dev/null | sed -e 's@.* @@')}
: ${COLUMNS:=80}
export COLUMNS

# Print text assuming it starts after other text in $startcol and
# needs to wrap before $COLUMNS - 2.  Subsequent lines start at $startcol.
# Long "words" will extend past $COLUMNS - 2.
fill_text() {
    local startcol="$1"
    local text="$2"

    local width=$(($COLUMNS - 2 - $startcol))
    [ $width -lt 0 ] && width=$((78 - $startcol))

    local out=""

    local padding
    if [ $startcol -gt 0 ] ; then
	padding=$(printf "\n%${startcol}s" " ")
    else
	padding="
"
    fi

    while [ -n "$text" ] ; do
	local orig="$text"

	# If we already have output then arrange padding on the next line.
	[ -n "$out" ] && out="${out}${padding}"

	# Break the text at $width.
	out="${out}${text:0:${width}}"
	text="${text:${width}}"

	# If we have left over text then the line break may be ugly,
	# so let's check and try to break it on a space.
	if [ -n "$text" ] ; then
	    # The 'x's stop us producing a special character like '(',
	    # ')' or '!'.  Yuck - there must be a better way.
	    if [ "x${text:0:1}" != "x " -a "x${text: -1:1}" != "x " ] ; then
		# We didn't break on a space.  Arrange for the
		# beginning of the broken "word" to appear on the next
		# line but not if it will make us loop infinitely.
		if [ "${orig}" != "${out##* }${text}" ] ; then
		    text="${out##* }${text}"
		    out="${out% *}"
		else
		    # Hmmm, doing that would make us loop, so add the
		    # rest of the word from the remainder of the text
		    # to this line and let it extend past $COLUMNS - 2.
		    out="${out}${text%% *}"
		    if [ "${text# *}" != "$text" ] ; then
			# Remember the text after the next space for next time.
			text="${text# *}"
		    else
			# No text after next space.
			text=""
		    fi
		fi
	    else
		# We broke on a space.  If it will be at the beginning
		# of the next line then remove it.
		text="${text# }"
	    fi
	fi
    done

    echo "$out"
}

# Display usage text, trying these approaches in order.
# 1. See if it all fits on one line before $COLUMNS - 2.
# 2. See if splitting before the default value and indenting it
#    to $startcol means that nothing passes $COLUMNS - 2.
# 3. Treat the message and default value as a string and just us fill_text()
#    to format it. 
usage_display_text () {
    local startcol="$1"
    local desc="$2"
    local default="$3"
    
    local width=$(($COLUMNS - 2 - $startcol))
    [ $width -lt 0 ] && width=$((78 - $startcol))

    default="(default \"$default\")"

    if [ $((${#desc} + 1 + ${#default})) -le $width ] ; then
	echo "${desc} ${default}"
    else
	local padding=$(printf "%${startcol}s" " ")

	if [ ${#desc} -lt $width -a ${#default} -lt $width ] ; then
	    echo "$desc"
	    echo "${padding}${default}"
	else
	    fill_text $startcol "${desc} ${default}"
	fi
    fi
}

# Display usage information for long config options.
usage_smart_display () {( # sub-shell for local declaration of defconf()
	local startcol=33

	defconf() {
	    local local longopt=$(echo "$1" | tr 'A-Z_' 'a-z-')

	    printf "     --%-25s " "${longopt}=${3}"

	    usage_display_text $startcol "$4" "$2"
	}

	"$@"
)}


# Display usage information for long config options.
usage_config_options (){
    usage_smart_display load_config
}

actions_init ()
{
    actions=""
}

actions_add ()
{
    actions="${actions}${actions:+ }$*"
}

actions_run ()
{
    [ -n "$actions" ] || usage

    local a
    for a in $actions ; do
	$a
    done
}

######################################################################

post_config_hooks=

######################################################################

load_config

############################
# parse command line options
long_opts=$(getopt_config_options)
getopt_output=$(getopt -n autocluster -o "c:e:E:xh" -l help,dump -l "$long_opts" -- "$@")
[ $? != 0 ] && usage

use_default_config=true

# We do 2 passes of the options.  The first time we just handle usage
# and check whether -c is being used.
eval set -- "$getopt_output"
while true ; do
    case "$1" in
	-c) shift 2 ; use_default_config=false ;;
	-e) shift 2 ;;
	-E) shift 2 ;;
	--) shift ; break ;;
	--dump|-x) shift ;;
	-h|--help) usage ;; # Usage should be shown here for real defaults.
	--*) shift 2 ;; # Assume other long opts are valid and take an arg.
	*) usage ;; # shouldn't happen, so this is reasonable.
    esac
done

config="./config"
$use_default_config && [ -r "$config" ] && . "$config"

eval set -- "$getopt_output"

while true ; do
    case "$1" in
	-c)
	    b=$(basename $2)
	    # force at least ./local_file to avoid accidental file
	    # from $PATH
	    . "$(dirname $2)/${b}"
	    # If $CLUSTER is unset then try to base it on the filename
	    if [ ! -n "$CLUSTER" ] ; then
		case "$b" in
		    *.autocluster)
			CLUSTER="${b%.autocluster}"
		esac
	    fi
	    shift 2
	    ;;
	-e) no_sanity=1 ; run_hooks post_config_hooks ; eval "$2" ; exit ;;
	-E) eval "$2" ; shift 2 ;;
	-x) set -x; shift ;;
	--dump) no_sanity=1 ; run_hooks post_config_hooks ; dump_config ;;
	--) shift ; break ;;
	-h|--help) usage ;; # Redundant.
	--*)
	    # Putting --opt1|opt2|... into a variable and having case
	    # match against it as a pattern doesn't work.  The | is
	    # part of shell syntax, so we need to do this.  Look away
	    # now to stop your eyes from bleeding! :-)
	    x=",${long_opts}" # Now each option is surrounded by , and :
	    if [ "$x" != "${x#*,${1#--}:}" ] ; then
		# Our option, $1, surrounded by , and : was in $x, so is legal.
		setconf_longopt "$1" "$2"; shift 2
	    else
		usage
	    fi
	    ;;
	*) usage ;; # shouldn't happen, so this is reasonable.
    esac
done

run_hooks post_config_hooks 

# catch errors
set -e
set -E
trap 'es=$?; 
      echo ERROR: failed in function \"${FUNCNAME}\" at line ${LINENO} of ${BASH_SOURCE[0]} with code $es; 
      exit $es' ERR

# check for needed programs 
check_command expect

[ $# -lt 1 ] && usage

t="$1"
shift

case "$t" in
    base)
	actions_init
	for t in "$@" ; do
	    case "$t" in
		create|boot) actions_add "base_${t}" ;;
		*) usage ;;
	    esac
	done
	actions_run
	;;

    cluster)
	actions_init
	for t in "$@" ; do
	    case "$t" in
		destroy|undefine|create|update_hosts|boot|setup)
		    actions_add "cluster_${t}" ;;
		build)
		    for t in destroy undefine create update_hosts boot setup ; do
			actions_add "cluster_${t}"
		    done
		    ;;
		*) usage ;;
	    esac
	done
	cluster_nodelist_hacking
	actions_run
	;;

    create)
	t="$1"
	shift
	case "$t" in
	    base)
		[ $# != 0 ] && usage
		base_create
		;;
	    cluster)
		[ $# != 1 ] && usage
		cluster_create "$1"
		;;
	    node)
		[ $# != 2 ] && usage
		create_one_node "$1" "$2"
		;;
	    *)
		usage;
		;;
	esac
	;;
    mount)
	[ $# != 1 ] && usage
	diskimage mount "$1"
	;;
    unmount|umount)
	[ $# != 0 ] && usage
	diskimage unmount
	;;
    bootbase)
	base_boot;
	;;
    *)
	usage;
	;;
esac