kvm-cluster

#!/bin/bash
#
# Operate on a virsh cluster
#
# Directory layout:
# <base>/			- Base directory for clusters.  Lots of storage space recommended
#       /BASEIMG/		- Directory containing the flavor directories
#       /BASEIMG/<flavor>	- Directory containing the image file and XML templates for <flavor>
#       /clusters/		- Directory containing instantiated cluster
#                /<cname>	- Cluster dir; contains the disk image, and XML used to create the cluster
#
# Commands:
# info	- Print info about the named cluster
# create	- Create a new cluster with the given name and parameters
# delete	- Delete an existing cluster.  All boxes and networks that can be found are nuked, and the cluster directory.
# rename	- Rename a cluster.  This invloves deleting the virsh config (but not the disk images) and re-creating, because virsh cannot rename
# reset		- Reset a cluster's storage back to default.
#
# Environment:
# KVM_CLUSTER_BASE	- This can hold the path to the base of the cluster tree (<base> above).  It defaults to PWD

if [ -n "${KVM_CLUSTER_BASE}" ]; then
	BASEDIR="${KVM_CLUSTER_BASE}"
else
	BASEDIR="${PWD}"
fi

CBASE="clusters"
FLAVOR="fedora"
NUMBOXES=2
NUMNETS=2
NUMVBLOCKS=1
VBLOCK_SIZE="20G"
VCPU="2"
VMEMGB="8"
VURI="qemu:///system"
FLAVORS="fedora rhel-atomic centos7"

# Set global usage output
DESCRIPTION="Control a KVM-based cluster"
USAGE="[-h |--help] [-V | --verbose] [-b <base-dir> | --base-directory=<base-dir>] [-c <cluster-dir> | --cluster-directory=<cluster-dir>] (info | create | delete | rename | reset | boot | halt)"
LONGUSAGE="Common options:
\t-h, --help\n\t\tPrint this help message
\t-V, --verbose\n\t\tBe verbose
\t-b <base-directory>, --base-directory=<base-directory>\n\t\tBase setup directory. Default: ${BASEDIR}
\t-c <cluster-dir>, --cluster-directory=<cluster-dir>\n\t\tDirectory for cluster. Default: ${BASEDIR}/${CBASE}/<cluster-name>

\t Commands:
\tinfo\n\t\tPrint info about a cluster
\tcreate\n\t\tCreate a new cluster
\tdelete\n\t\tDelete an existing cluster
\trename\n\t\tRename an existing cluster
\treset\n\t\tReset an existing cluster
\tboot\n\t\tBoot any boxes in the cluster not currently running
\thalt\n\t\tHalt any boxes in the cluster currently running

\tFor more info about each command, pass --help to that command

Known flavors: ${FLAVORS}
"

# Standard functions
# if is_interactive; then echo "interactive" fi
#
# Check for an interactive shell
is_interactive() {
	case $- in
		*i*)
			# Don't die in interactive shells
			return 0
			;;
		*)
			return 1
			;;
	esac
}

# if can_die; then exit
#
# Check to see if it's legal to exit during die
can_die() {
	if (( BASH_SUBSHELL > 0 )); then
		return 0
	fi
	if ! is_interactive; then
		return 0
	fi
	return 1
}

# In a function:
# command || die "message" || return 1
# Outside a function:
# command || die "message"
#
# Print a message and exit with failure
die() {
	echo -e "Failed: $1" >&2
	if can_die; then
		exit 1
	fi
	return 1
}

# In a function:
# command || die "message" "cleanupfunc" <cleanup args> || return 1
# Outside a function:
# command || die "message" "cleanupfunc" <cleanup args>
#
# Print a message and exit with failure
die_cleanup() {
	local message=$1
	local cleanfunc=$1
	shift 2
	if [ ! -z "$(declare -F | grep "${cleanfunc}")" ]; then
		eval ${cleanfunc} "$@"
	fi
	return $(die ${message})
}

usage() {
	local myusage;
	if [ -n "${USAGE}" ]; then
		myusage=${USAGE}
	else
		myusage="No usage given"
	fi
	local me;
	if [ -n "${ME}" ]; then
		me=${ME}
	else
		me=$(basename $0)
	fi
	if [ -n "$1" ]; then
		echo "$@"
	fi
	echo ""
	if [ -n "${DESCRIPTION}" ]; then
		echo -e "${me}: ${DESCRIPTION}"
		echo ""
	fi
	echo "Usage:"
	echo "${me} ${myusage}"
	if [ -n "${LONGUSAGE}" ]; then
		echo -e "${LONGUSAGE}"
	fi
	exit 1
}

# Script name
ME=$(basename $0)

# Parse global arguments
ARGS=`getopt -o +hVb:c: --long help,verbose,base-directory:,cluster-directory: -n "${ME}" -- "$@"`

if [ $? != 0 ] ; then
	usage "invalid arguments"
fi
eval set -- "$ARGS"

while true ; do
	case "$1" in
		-h|--help) usage; shift ;;
		-V|--verbose) export GTWS_VERBOSE="yes"; shift ;;
		-b|--base-directory) BASEDIR=$2 ; shift 2 ;;
		-c|--cluster-directory) CDIR=$2 ; shift 2 ;;
		--) shift ; break ;;
		* ) usage "Invalid argument $1" ;;
	esac
done

#
# Opts processing for sub-commands
#

opts_info() {
	local ME="${ME} ${COMMAND}"
	local USAGE="[-h |--help] <cluster-name>"
	local LONGUSAGE="\t-h, --help\n\t\tPrint this help message
\t<cluster-name>\n\t\tName of cluster about which to get info.

\tPrint info about a cluster, including boot state.
"
	local SUBARGS=""
	SUBARGS=`getopt -o h --long help -n "${ME}" -- "$@"`
	if [ $? != 0 ] ; then
		usage "invalid arguments"
	fi
	eval set -- "$SUBARGS"

	while true ; do
		case "$1" in
			-h|--help) usage; shift ;;
			--) shift ; break ;;
			* ) usage "Invalid argument $1" ;;
		esac
	done
	CNAME="${1}"
	if [ -z "${CNAME}" ]; then
		usage "Must give cluster name"
	fi
}

opts_create() {
	local ME="${ME} ${COMMAND}"
	local USAGE="[-h |--help] [-c <number-cpus> | --cpu-count=<number-cpus>] [-f <flavor> | --flavor=<flavor>] [-i <image-directory> | --image-directory=<image-directory>] [-m <number-machines> | --machine-count=<number-machines>] [-r <ram-GB> | --ram-size=<ram-GB>] [-n <number-networks> | --network-count=<number-networks>] [-v <number-vblock> | --vblock-count=<number-vblock>] <cluster-name>"
	local LONGUSAGE="\t-h, --help\n\t\tPrint this help message
\t-c <number-cpus>, --cpu-count=<number-cpus>\n\t\tNumber of virtual CPUs per machine. Default: ${VCPU}
\t-f <flavor>, --flavor=<flavor>\n\t\tFlavor to install.  Default: ${FLAVOR}
\t-i <image-directory>, --image-directory=<image-directory>\n\t\tDirectory containing image flavors. Default: ${BASEDIR}/BASEDIR
\t-m <number-machines>, --machine-count=<number-machines>\n\t\tNumber of machines in cluster. Default: ${NUMBOXES}
\t-n <number-networks>, --network-count=<number-networks>\n\t\tNumber of networks in cluster. Default: ${NUMNETS}
\t-r <ram-GB>, --ram-size=<ram-GB>\n\t\tSize of RAM per box in GB. Default: ${VMEMGB}
\t-v <number-vblock>, --vblock-count=<number-vblock>\n\t\tNumber of extra virtual block devices. Default: ${NUMVBLOCKS}
\t<cluster-name>\n\t\tName of cluster to create.

\tCreate a new cluster.  All clusters are on the public bridged network, and have a boot block device (based on the base image).  The number of boxes, private networks, and extra block devices can be configured.
"
	local SUBARGS=""
	SUBARGS=`getopt -o hc:f:i:m:n:r:v: --long help,cpu-count:,flavor:,image-directory:,machine-count:,network-count:,ram-size:,vblock-count: -n "${ME}" -- "$@"`
	if [ $? != 0 ] ; then
		usage "invalid arguments"
	fi
	eval set -- "$SUBARGS"

	while true ; do
		case "$1" in
			-h|--help) usage; shift ;;
			-c|--cpu-count) VCPU=$2 ; shift 2 ;;
			-f|--flavor) FLAVOR=$2 ; shift 2 ;;
			-i|--image-directory) IDIR=$2 ; shift 2 ;;
			-m|--machine-count) NUMBOXES=$2 ; shift 2 ;;
			-n|--network-count) NUMNETS=$2 ; shift 2 ;;
			-r|--ram-size) VMEMGB=$2 ; shift 2 ;;
			-v|--vblock-count) NUMVBLOCKS=$2 ; shift 2 ;;
			--) shift ; break ;;
			* ) usage "Invalid argument $1" ;;
		esac
	done
	CNAME="${1}"
	if [ -z "${CNAME}" ]; then
		die "Must give cluster name"
	fi
}

opts_delete() {
	local ME="${ME} ${COMMAND}"
	local USAGE="[-h |--help] <cluster-name>"
	local LONGUSAGE="\t-h, --help\n\t\tPrint this help message
\t<cluster-name>\n\t\tName of cluster to delete

\tDelete a cluster
"
	local SUBARGS=""
	SUBARGS=`getopt -o h --long help -n "${ME}" -- "$@"`
	if [ $? != 0 ] ; then
		usage "invalid arguments"
	fi
	eval set -- "$SUBARGS"

	while true ; do
		case "$1" in
			-h|--help) usage; shift ;;
			--) shift ; break ;;
			* ) usage "Invalid argument $1" ;;
		esac
	done
	CNAME="${1}"
	if [ -z "${CNAME}" ]; then
		die "Must give cluster name"
	fi
}

opts_rename() {
	local ME="${ME} ${COMMAND}"
	local USAGE="[-h |--help] <old-cluster-name> <new-cluster-name>"
	local LONGUSAGE="\t-h, --help\n\t\tPrint this help message
\t<old-cluster-name>\n\t\tName of cluster to rename
\t<new-cluster-name>\n\t\tNew name for cluster

\tRename a cluster
"
	local SUBARGS=""
	SUBARGS=`getopt -o h --long help -n "${ME}" -- "$@"`
	if [ $? != 0 ] ; then
		usage "invalid arguments"
	fi
	eval set -- "$SUBARGS"

	while true ; do
		case "$1" in
			-h|--help) usage; shift ;;
			--) shift ; break ;;
			* ) usage "Invalid argument $1" ;;
		esac
	done
	CNAME="${1}"
	NEWCNAME="${2}"
	if [ -z "${NEWCNAME}" ]; then
		die "Must give old an new cluster names"
	fi
}

opts_reset() {
	local ME="${ME} ${COMMAND}"
	local USAGE="[-h |--help] [-n | --no-root] [-v | --vblock ] <cluster-name>"
	local LONGUSAGE="\t-h, --help\n\t\tPrint this help message
\t-n, --no-root\n\t\tDon't reset root block device
\t-v, --vblock\n\t\tReset virtual block devices as well
\t<cluster-name>\n\t\tName of cluster to reset.

\tReset a cluster
"
	local SUBARGS=""
	SUBARGS=`getopt -o hvn --long help,vblock,no-root -n "${ME}" -- "$@"`
	if [ $? != 0 ] ; then
		usage "invalid arguments"
	fi
	eval set -- "$SUBARGS"

	while true ; do
		case "$1" in
			-h|--help) usage; shift ;;
			-n|--no-root) NOROOT="true"; shift ;;
			-v|--vblock) RSETVBLOCK="true"; shift ;;
			--) shift ; break ;;
			* ) usage "Invalid argument $1" ;;
		esac
	done
	CNAME="${1}"
	if [ -z "${CNAME}" ]; then
		die "Must give cluster name"
	fi
}

opts_boot() {
	local ME="${ME} ${COMMAND}"
	local USAGE="[-h |--help] <cluster-name>"
	local LONGUSAGE="\t-h, --help\n\t\tPrint this help message
\t<cluster-name>\n\t\tName of cluster to boot

\tBoot any boxes in a cluster not already running.  Virtual networks are brought up, if they're not already up.
"
	local SUBARGS=""
	SUBARGS=`getopt -o h --long help -n "${ME}" -- "$@"`
	if [ $? != 0 ] ; then
		usage "invalid arguments"
	fi
	eval set -- "$SUBARGS"

	while true ; do
		case "$1" in
			-h|--help) usage; shift ;;
			--) shift ; break ;;
			* ) usage "Invalid argument $1" ;;
		esac
	done
	CNAME="${1}"
	if [ -z "${CNAME}" ]; then
		die "Must give cluster name"
	fi
}

opts_halt() {
	local ME="${ME} ${COMMAND}"
	local USAGE="[-h |--help] [-f |--force] <cluster-name>"
	local LONGUSAGE="\t-h, --help\n\t\tPrint this help message
\t-f | --force\n\t\tShut down paused boxes as well
\t<cluster-name>\n\t\tName of cluster to halt

\tHalt any boxes in a cluster that are running, and tear down private networks
"
	local SUBARGS=""
	SUBARGS=`getopt -o hf --long help,force -n "${ME}" -- "$@"`
	if [ $? != 0 ] ; then
		usage "invalid arguments"
	fi
	eval set -- "$SUBARGS"

	while true ; do
		case "$1" in
			-h|--help) usage; shift ;;
			-f|--force) FORCE="true"; shift ;;
			--) shift ; break ;;
			* ) usage "Invalid argument $1" ;;
		esac
	done
	CNAME="${1}"
	if [ -z "${CNAME}" ]; then
		die "Must give cluster name"
	fi
}

# Now get the subcommand.  It will be in $1.
COMMAND=$1; shift
case "${COMMAND}" in
	"create")	opts_create "$@" ;;
	"delete")	opts_delete "$@" ;;
	"rename")	opts_rename "$@" ;;
	"reset")	opts_reset "$@" ;;
	"info")		opts_info "$@" ;;
	"boot")		opts_boot "$@" ;;
	"halt")		opts_halt "$@" ;;
	"") usage "Must give a command" ;;
	* ) usage "Unknown command ${COMMAND}" ;;
esac

# getvolgroup ${flavor} volgrp
#
# Get the volume group name for the given flavor
#
# Result will be in the local variable volgrp  Or:
#
# volgrp=$(getvolgroup ${flavor})
#
function getvolgroup {
	local flavor=$1
	local __resultvar=$2
	local volgrp=""

	case $1 in
		"fedora") volgrp="vfedora" ;;
		"rhel-atomic") volgrp="rhelah" ;;
		"centos7") volgrp="centos" ;;
		*) volgrp="unknown" ;;
	esac

	if [[ "$__resultvar" ]]; then
		eval $__resultvar="'$volgrp'"
	else
		echo "$volgrp"
	fi
}

function getloop {
	local imgfile=$1
	local  __resultvar=$2
	local loop=$(losetup -l  | grep "${imgfile}" | awk '{print $1}')
	if [[ "$__resultvar" ]]; then
		eval $__resultvar="'$loop'"
	else
		echo "$loop"
	fi
}

function kvm_mount_cleanup {
	local imgfile=${1}
	local volgrp=${2}
	local cdir=$(dirname ${imgfile})
	local loop=$(getloop "${imgfile}")

	sudo umount "${cdir}/mnt"
	if [ -n "${volgrp}" ]; then
		sudo vgchange -an "${volgrp}" > /dev/null
	fi

	# Make sure vgs are cleaned up
	sudo dmsetup remove_all

	if [ -n "${loop}" ]; then
		sudo kpartx -d ${loop} > /dev/null
		sudo losetup -d ${loop}
	fi
	sudo rmdir "${cdir}/mnt"
}

function kvm_mount {
	local imgfile=${1}
	local volgrp=${2}
	local cdir=$(dirname ${imgfile})
	local loop=$(getloop "${imgfile}")
	if [ -n "${loop}" ]; then
		die "${imgfile} is already mounted" || return 1
	fi
	loop=$(sudo losetup -f)
	if [ -z "${loop}" ]; then
		die "Cannot get a loop device" || return 1
	fi

	sudo losetup ${loop} "${imgfile}" || die "Failed to setup loop for ${imgfile}" || return 1
	sleep 1

	sudo kpartx -a ${loop} || die_cleanup "Failed to initialize partition table for ${loop}" kvm_mount_cleanup ${imgfile} || return 1
	sleep 1

	sudo vgscan > /dev/null || die_cleanup "vgscan failed" kvm_mount_cleanup ${imgfile} || return 1
	sleep 1

	sudo vgchange -a y "${volgrp}" > /dev/null # Don't die - VG is checked below, and could fail for many reasons
	sleep 1

	sudo vgdisplay -s | grep "\"${volgrp}\"" || die_cleanup "Cannot find volume group ${volgrp}" kvm_mount_cleanup ${imgfile} || return 1

	if [ ! -d "${cdir}/mnt" ]; then
		sudo mkdir "${cdir}/mnt"
	fi

	sudo mount "/dev/${volgrp}/root" "${cdir}/mnt" || die_cleanup "Failed to mount root" kvm_mount_cleanup ${imgfile} ${volgrp} || return 1
}

function kvm_umount {
	# Use the cleanup function to avoid drift
	kvm_mount_cleanup ${1} ${2}
}

function create_image {
	local srcimg="${1}"
	local dstimg="${2}"
	local volgrp="${3}"
	local basename="${4}"
	local cname="${5}"
	local bnum="${6}"
	local cbytes="${7}"
	local cdir=$(dirname ${dstimg})

	cp --sparse=always "${srcimg}" "${dstimg}" || die "Failed to copy image"
	kvm_mount "${dstimg}" "${volgrp}" || die "Failed to mount image" || return 1
	sudo sed -i -e "s/${basename}/${cname}-${bnum}/" "${cdir}/mnt/etc/hostname"
	sudo sed -i -e "s/HWADDR=.*$/HWADDR=52:54:00${cbytes}:${bnum}/" "${cdir}/mnt/etc/sysconfig/network-scripts/ifcfg-eth0"
	kvm_umount "${dstimg}" "${volgrp}" || die "Failed to mount image" || return 1
}

pinfo_detail() {
	local cdir="${CDIR}"
	if [ ! -d "${CDIR}" ]; then
		cdir="${CDIR} (does not exist)"
	fi
	local flavor="unknown"
	if [ -f "${CDIR}/config" ]; then
		source "${CDIR}/config"
	fi
	echo "$@ cluster ${CNAME}:"
	echo -e "\tFlavor: ${flavor}"
	echo -e "\tDirectory: ${cdir}"
	echo -e "\tBoxes:"
	for box in ${VIRSH_BOXES}; do
		echo -e "\t\t${box}\t\t\t${BOX_STATE[${box}]}"
	done
	echo -e "\tNetworks:"
	for net in ${VIRSH_NETS}; do
		uuid=$(${VIRSH} net-info ${net} | grep UUID)
		echo -e "\t\tName:\t${net}"
		echo -e "\t\t${uuid}"
		echo -e "\t\tActive:\t\t${NET_STATE[${net}]}"
	done
}
pinfo() {
	local cdir="${CDIR}"
	if [ ! -d "${CDIR}" ]; then
		cdir="${CDIR} (does not exist)"
	fi
	local flavor="unknown"
	if [ -f "${CDIR}/config" ]; then
		source "${CDIR}/config"
	fi
	echo "$@ cluster ${CNAME}:"
	echo -e "\tFlavor: ${flavor}"
	echo -e "\tBoxes: ${VIRSH_BOXES}"
	echo -e "\tNetworks: ${VIRSH_NETS}"
}
cmd_exists() {
	which $1 > /dev/null 2>&1
	if [ "$?" == "1" ]; then
		die "You don't have $1 installed, sorry"
	fi
}

# Command functions
cmd_reset() {
	if [ ! -d "${CDIR}" ]; then
		die "Cannot reset cluster ${CNAME}: directory does not exist"
	elif [ -z "${VIRSH_BOXES}" ]; then
		die "Cannot reset cluster ${CNAME}: No VMs exist"
	elif [ ! -d "${IDIR}" ]; then
		die "Cannot reset cluster ${CNAME}: No image base directory ${IDIR}"
	elif [ ! -f "${CDIR}/config" ]; then
		die "Cannot reset cluster ${CNAME}: No config"
	fi

	# Source env to get FLAVOR and CBYTES
	source "${CDIR}/config"
	if [ -z "${FLAVOR}" ]; then
		die "Cannot reset cluster ${CNAME}: flavor not in config"
	elif [ -z "${CBYTES}" ]; then
		die "Cannot reset cluster ${CNAME}: CBYTES not in config"
	fi

	local boxarr=( ${VIRSH_BOXES} )
	NUMBOXES=${#boxarr[@]}

	echo "Reseting cluster ${CNAME}:"
	echo -e "\tDirectory: ${CDIR}"
	echo -e "\tImage Directroy: ${IDIR}"
	echo -e "\tFlavor: ${FLAVOR}"
	echo -e "\tNumber of boxes: ${NUMBOXES}"
	echo -e "\tNumber of networks: ${NUMNETS}"
	echo -e "\tNumber of virtual block devices: ${NUMVBLOCKS}"
	echo -e "\tMAC bytes: ${CBYTES}"

	# Power them all down
	for box in ${VIRSH_BOXES}; do
		${VIRSH} destroy ${box} > /dev/null 2>&1
	done


	local basename="${FLAVOR}-base"
	local volgrp=$(getvolgroup "${FLAVOR}")

	for bnum in $(seq -f '%02.0f' ${NUMBOXES}); do
		echo "Resetting box ${bnum}"
		if [ -z "${NOROOT}" ]; then
			sudo rm "${CDIR}/${CNAME}-box${bnum}.img"
			create_image "${IDIR}/${FLAVOR}/box.img" "${CDIR}/${CNAME}-box${bnum}.img" "${volgrp}" "${basename}" "${CNAME}" "${bnum}" "${CBYTES}" || die "failed to mangle image"
		fi
		if [ -n "${RSETVBLOCK}" ]; then
			for vblock in $(seq -f '%02.0f' ${NUMVBLOCKS}); do
				sudo rm "${CDIR}/${CNAME}-box${bnum}-vblock${vblock}.img"
				qemu-img create -f raw "${CDIR}/${CNAME}-box${bnum}-vblock${vblock}.img" "${VBLOCK_SIZE}"
			done
		fi
	done
}

cmd_create() {
	if [ -d "${CDIR}" ]; then
		die "Cannot create cluster ${CNAME}: directory exists"
	elif [ -n "${VIRSH_BOXES}" ]; then
		die "Cannot create cluster ${CNAME}: VMs exist"
	elif [ -n "${VIRSH_NETR}" ]; then
		die "Cannot create cluster ${CNAME}: networks exist"
	elif [ ! -d "${IDIR}" ]; then
		die "Cannot create cluster ${CNAME}: No image base directory ${IDIR}"
	fi

	local cbytes=$(hexdump -n2 -e '/1 ":%02X"' /dev/urandom)

	echo "Creating cluster ${CNAME}:"
	echo -e "\tDirectory: ${CDIR}"
	echo -e "\tImage Directroy: ${IDIR}"
	echo -e "\tFlavor: ${FLAVOR}"
	echo -e "\tNumber of boxes: ${NUMBOXES}"
	echo -e "\tNumber of virtual block devices: ${NUMVBLOCKS}"
	echo -e "\tMAC bytes: ${cbytes}"

	local basename="${FLAVOR}-base"
	local volgrp=$(getvolgroup "${FLAVOR}")

	mkdir -p "${CDIR}" || die "failed to create ${CDIR}"

	echo "FLAVOR=${FLAVOR}" > "${CDIR}/config"
	echo "CBYTES=${cbytes}" >> "${CDIR}/config"
	echo "NUMVBLOCKS=${NUMVBLOCKS}" >> "${CDIR}/config"

	for net in $(seq -f '%02.0f' 1 ${NUMNETS}); do
		NNAME="${CDIR}/${CNAME}-net${net}.xml"
		cp "${IDIR}/${FLAVOR}/net-template.xml" "${NNAME}" || die "Failed to copy XML template for ${net}"
		NETBYTE=$((${net} + 20))
		sed -i -e "s#CLUSTERNAME#${CNAME}#" -e "s#NETNUM#${net}#" -e "s#NETBYTE#${NETBYTE}#" -e "s#CBYTES#${cbytes}#" "${NNAME}"

		${VIRSH} net-define "${NNAME}" || die "Couldn't create ${NNAME}"
		#${VIRSH} net-start "${CNAME}-n${net}" || die "Couldn't start ${CNAME}-n${net}"
	done

	for bnum in $(seq -f '%02.0f' ${NUMBOXES}); do
		create_image "${IDIR}/${FLAVOR}/box.img" "${CDIR}/${CNAME}-box${bnum}.img" "${volgrp}" "${basename}" "${CNAME}" "${bnum}" "${cbytes}" || die "failed to mangle image"
		DNAME="${CDIR}/${CNAME}-box${bnum}.xml"
		cp "${IDIR}/${FLAVOR}/box-template.xml" "${DNAME}" || die "Failed to copy XML template for ${bnum}"

		IFACES=""
		for net in $(seq -f '%02.0f' ${NUMNETS}); do
			SLOTNUM=$((${net} + 10))
			NIFACE=$(sed -e "s#NETNUM#${net}#" -e "s#CLUSTERNAME#${CNAME}#" -e "s#BNUM#${bnum}#" -e "s#SLOTNUM#${SLOTNUM}#" -e "s#CBYTES#${cbytes}#" "${IDIR}/${FLAVOR}/iface-template.xml" | sed '{:q;N;s/\n/\\n/g;t q}')
			IFACES+="${NIFACE}\n"
		done

		local vblocks=""
		local vdev="a"
		for vblock in $(seq -f '%02.0f' ${NUMVBLOCKS}); do
			SLOTNUM=$((${vblock} + 20))
			vdev=$(echo "${vdev}" | tr "a-z" "b-z")
			NVBLOCK=$(sed -e "s#VNUM#${vblock}#" -e "s#CLUSTERNAME#${CNAME}#" -e "s#CDIR#$CDIR#" -e "s#BNUM#${bnum}#" -e "s#SLOTNUM#${SLOTNUM}#" -e "s#VDEV#sd${vdev}#" "${IDIR}/${FLAVOR}/vblock-template.xml" | sed '{:q;N;s/\n/\\n/g;t q}')
			vblocks+="${NVBLOCK}\n"
			qemu-img create -f raw "${CDIR}/${CNAME}-box${bnum}-vblock${vblock}.img" "${VBLOCK_SIZE}"
		done

		VMEM=$((${VMEMGB} * 1024 * 1024))
		sed -i -e "s#CLUSTERNAME#${CNAME}#" -e "s#BNUM#${bnum}#" -e "s#CDIR#$CDIR#" -e "s#CBYTES#${cbytes}#" -e "s#IFACES#${IFACES}#" -e "s#VBLOCKS#${vblocks}#" -e "s#VCPU#${VCPU}#" -e "s#VMEM#${VMEM}#" ${DNAME}
		${VIRSH} define "${DNAME}" || die "Couldn't define ${DNAME}"
	done

	echo "Cluster ${CNAME} complete."
}

cmd_delete() {
	# Always try and delete cluster, so left-over parts can be nuked
	pinfo "Deleting"

	for box in ${VIRSH_BOXES}; do
		${VIRSH} destroy ${box} > /dev/null 2>&1
		${VIRSH} undefine ${box} || die "Failed to undefine ${box}"
	done
	for net in ${VIRSH_NETS}; do
		${VIRSH} net-destroy ${net} > /dev/null 2>&1 # May not have been active
		${VIRSH} net-undefine ${net}  > /dev/null 2>&1 # may have been non-persistant
	done
	if [ -d "${CDIR}" ]; then
		rm -rf "${CDIR}" || die "Failed to remove ${CDIR}"
	fi
	exit
}

cmd_rename() {
	if [ -z "${NEWCNAME}" ]; then
		usage "--rename requires a new cluster name"
	elif [ ! -d "${CDIR}" ]; then
		die "cannot rename cluster ${CNAME}: directory does not exist"
	elif [ -z "${VIRSH_BOXES}" ]; then
		die "cannot rename cluster ${CNAME}: VMs do not exist"
	fi
	echo "Renaming cluster ${CNAME} to ${NEWNAME}"
	pinfo "Original"

	TDIR=$(mktemp -d)

	# Power them all down
	for box in ${VIRSH_BOXES}; do
		${VIRSH} destroy ${box} > /dev/null 2>&1
	done

	# Move dir, rename files
	newdir=${CDIR/${CNAME}/${NEWCNAME}}
	sudo mv "${CDIR}" "${newdir}"

	# Rename the neworks
	for net in ${VIRSH_NETS}; do
		if [[ $net =~ pnet ]]; then
			newnet=${net/${CNAME}-pnet/${NEWCNAME}-n}
			oldfbase=${net/${CNAME}-pnet/${CNAME}-net}
			newfbase=${newnet/${NEWCNAME}-pnet/${NEWCNAME}-net}
		else
			newnet=${net/${CNAME}/${NEWCNAME}}
			oldfbase=${net/${CNAME}-n/${CNAME}-net}
			newfbase=${newnet/${NEWCNAME}-n/${NEWCNAME}-net}
		fi
		${VIRSH} net-dumpxml ${net} > "${TDIR}/${newnet}.xml" || die "Failed to dump ${net}"
		${VIRSH} net-destroy ${net} || die "Failed to destroy ${net}"
		${VIRSH} net-undefine ${net}  > /dev/null 2>&1 # may have been non-persistant
		sudo mv "${newdir}/${oldfbase}.xml" "${newdir}/${newfbase}.xml"
		for f in "${TDIR}/${newnet}.xml" "${newdir}/${newfbase}.xml"; do
			sed -i  -e "s#<name>${net}</name>#<name>${newnet}</name>#g" \
				-e "s#virbr-${CNAME}#virbr-${NEWCNAME}#g" \
				-e "s#pnet#n#g" \
				"${f}"
		done
		${VIRSH} net-define "${TDIR}/${newnet}.xml" || die "Couldn't create ${newnet}"
		${VIRSH} net-start "${newnet}" || die "Couldn't start ${newnet}"
	done

	# Rename boxes
	for box in ${VIRSH_BOXES}; do
		newbox=${box/${CNAME}/${NEWCNAME}}
		oldfbase=${box/${CNAME}-/${CNAME}-box}
		newfbase=${newbox/${NEWCNAME}-/${NEWCNAME}-box}
		${VIRSH} dumpxml ${box} > "${TDIR}/${newbox}.xml" || die "Failed to dump ${box}"
		${VIRSH} undefine ${box} || die "Failed to destroy ${box}"
		sudo mv "${newdir}/${oldfbase}.img" "${newdir}/${newfbase}.img"
		sudo mv "${newdir}/${oldfbase}.xml" "${newdir}/${newfbase}.xml"
		for f in "${TDIR}/${newbox}.xml" "${newdir}/${newfbase}.xml"; do
			sed -i  -e "s#<name>${box}</name>#<name>${newbox}</name>#g" \
				-e "s#<source file='${CDIR}/${oldfbase}#<source file='${newdir}/${newfbase}#g" \
				-e "s#<source network='${CNAME}#<source network='${NEWCNAME}#g" \
				-e "s#pnet#n#g" \
				"${f}"
		done
		${VIRSH} define "${TDIR}/${newbox}.xml" || die "Couldn't create ${newbox}"
	done

	rm -rf "${TDIR}"

	echo "Renamed cluster ${CNAME} to ${NEWCNAME}"
}

cmd_info() {
	pinfo_detail "Info on"
}

cmd_boot() {
	if [ -z "${VIRSH_BOXES}" ]; then
		die "cannot boot cluster ${CNAME}: VMs do not exist"
	fi
	pinfo "Booting"
	for net in ${VIRSH_NETS}; do
		if [ "${NET_STATE[${net}]}" == "no" ]; then
			${VIRSH} net-start ${net} || die "Failed to start ${net}"
		fi
	done
	for box in ${VIRSH_BOXES}; do
		if [ "${BOX_STATE[${box}]}" == "halted" ]; then
			${VIRSH} start ${box} || die "Failed to boot ${box}"
		fi
	done
}

cmd_halt() {
	if [ -z "${VIRSH_BOXES}" ]; then
		die "cannot halt cluster ${CNAME}: VMs do not exist"
	fi
	pinfo "Halting"
	for box in ${VIRSH_BOXES}; do
		if [ "${BOX_STATE[${box}]}" == "running" ]; then
			${VIRSH} shutdown ${box} || die "Failed to halt ${box}"
		fi
	done
	if [ -n "${FORCE}" ]; then
		echo "Force Halting"
		for box in ${VIRSH_BOXES}; do
			if [ "${BOX_STATE[${box}]}" == "paused" ]; then
				${VIRSH} destroy ${box} || die "Failed to halt ${box}"
			fi
		done
	fi
	for net in ${VIRSH_NETS}; do
		if [ "${NET_STATE[${net}]}" == "yes" ]; then
			${VIRSH} net-destroy ${net} || die "Failed to stop ${net}"
		fi
	done
}

if [ -z "${CDIR}" ]; then
	CDIR="${BASEDIR}/${CBASE}/${CNAME}"
fi
if [ -z "${IDIR}" ]; then
	IDIR="${BASEDIR}/BASEDIR"
fi

cmd_exists virsh
VIRSH="virsh -c ${VURI}"

#IFS=', ' read -a VIRSH_BOXES <<< "$string"
VIRSH_BOXES=$(${VIRSH} list --all | egrep "\<${CNAME}-[0-9]" | awk '{printf "%s ", $2}')
running=$(${VIRSH} list --state-running | egrep "\<${CNAME}-[0-9]" | awk '{printf "%s ", $2}')
paused=$(${VIRSH} list --state-paused | egrep "\<${CNAME}-[0-9]" | awk '{printf "%s ", $2}')
declare -A BOX_STATE
for box in ${VIRSH_BOXES}; do
	is_running="halted"
	for run in ${running}; do
		if [ "${run}" == "${box}" ]; then
			is_running="running"
			break
		fi
	done
	for run in ${paused}; do
		if [ "${run}" == "${box}" ]; then
			is_running="paused"
			break
		fi
	done
	BOX_STATE[$box]=${is_running}
done
VIRSH_NETS=$(${VIRSH} net-list --all | egrep "(\<${CNAME}-pnet[0-9])|(\<${CNAME}-n[0-9])" | awk '{printf "%s ", $1}')
running=$(${VIRSH} net-list | egrep "(\<${CNAME}-pnet[0-9])|(\<${CNAME}-n[0-9])" | awk '{printf "%s ", $1}')
declare -A NET_STATE
for net in ${VIRSH_NETS}; do
	is_running="no"
	for run in ${running}; do
		if [ "${run}" == "${net}" ]; then
			is_running="yes"
			break
		fi
	done
	NET_STATE[$net]=${is_running}
done

# Run command
case "${COMMAND}" in
	"create")	cmd_create ;;
	"delete")	cmd_delete ;;
	"rename")	cmd_rename ;;
	"reset")	cmd_reset ;;
	"info")		cmd_info ;;
	"boot")		cmd_boot ;;
	"halt")		cmd_halt ;;
	* ) usage "Unknown command ${COMMAND}";;
esac