path: root/packager/openslx.functions

# ------------------------------------------------------------------------------
#
#                          Functions for OpenSLX-NG
#
# ------------------------------------------------------------------------------
#
# check if we have our environment variables to check
# if we actually got sourced by the main script
if [ -z "${SELF_PID}" -o  -z "${ROOT_DIR}" ]; then
	# not using perror, since we probably don't have it.
	echo "Neither SELF_PID nor ROOT_DIR is set. Was this included by OpenSLX-NG?"
	exit 1
fi

# ------------------------------------------------------------------------------
#
#                          General helper functions
#
# ------------------------------------------------------------------------------
banner () {
	echo -e "\033[38;5;196m\t"'                                    .__          '
	echo -e "\033[38;5;196m\t"'  ____ ______   ____   ____   _____|  | ___  ___ '
	echo -e "\033[38;5;202m\t"' /  _ \\\\____ \\_/ __ \\ /    \\ /  ___/  | \\  \\/  / '
	echo -e "\033[38;5;208m\t"'(  <_> )  |_> >  ___/|   |  \\\\___ \\|  |__>    <  '
	echo -e "\033[38;5;214m\t"' \\____/|   __/ \\___  >___|  /____  >____/__/\\_ \\ '
	echo -e "\033[38;5;220m\t"'       |__|        \\/     \\/     \\/           \\/ '
	echo -e "\033[38;5;220m\t"
	echo -e "\033[38;5;226m\t                     ** OpenSLX Project // 2015 **"
	echo -e "\033[38;5;232m\t                        http://lab.openslx.org/"
	echo -e "\033[00m"
}

pinfo() {
	echo -e "\033[38;5;10m[info]\033[0m $@"
}

pwarning() {
	echo -e "\033[38;5;11m[warning]\033[0m $@"
}

perror() {
	echo -e "\033[38;5;9m[error]\033[0m $@"
	kill "$SELF_PID"
	exit 1
}

# accepts one argument to be printed as a warning before dumping the help
print_usage() {
	[ -n "$1" ] && pwarning "$1"
	pinfo "USAGE:"
	pinfo "$ARG0 <action> <actions_params>"
	pinfo "\t\tActions: '--clone', '--package', '--export', 'update'"
	pinfo ""
	pinfo "CLONING:      rsync remote host to local directory"
	pinfo "$ARG0 --clone --host <host> [--syncdir <path>]"
	pinfo "\t\tIf not specified, --syncdir = './clones/<host>/stage4'"
	pinfo ""
	pinfo "PACKAGING:    pack local rsync directory as qcow2-container"
	pinfo "$ARG0 --package --host <host> [--container <path>] [--syncdir <dir>]"
	pinfo "\t\tIf <container> is not specified, --container = './clones/<host>/stage4.qcow2'"
	pinfo "\t\tIf <syncdir> is not specified, --syncdir = './clones/<host>/stage4'"
	pinfo ""
	pinfo "EXPORTING:    rsync remote host to a new qcow2-container"
	pinfo "$ARG0 --export --host <host> [--container <path>] [--syncdir <dir>]"
	pinfo "\t\tIf <container> is not specified, --container = './clones/<host>/stage4.qcow2'"
	pinfo ""
	pinfo "UPDATING:     rsync remote host to an existing qcow2-container"
	pinfo "$ARG0 --update --host <host> [--container <path>]"
	pinfo "\t\tIf <container> is not specified, --container = './clones/<host>/stage4.qcow2'"
	kill "$SELF_PID"
	exit 1
}

# helper to parse the command line arguments and fill the environment
# with the parameters given. Since the fallbacks for those variables
# are only dependent on the selected action, we will also post-process
# them to make sure they are set when leaving this function!
read_params() {
	# initialize global variables
	declare -g FORCE=0
	unset ACTION REMOTE_HOST CONTAINER_PATH RSYNC_TARGET

	# handle rest of arguments
	while [ "$#" -gt "0" ]; do
		local PARAM="$1"
		shift

		# options to current target
		if [[ "$PARAM" == --* ]]; then
			case "$PARAM" in
				--clone)
					declare -rg ACTION="CLONE"
					;;
				--package)
					declare -rg ACTION="PACKAGE"
					;;
				--export)
					declare -rg ACTION="EXPORT"
					;;
				--update)
					declare -rg ACTION="UPDATE"
					;;
				--host)
					if [ -z "$1" ]; then 
						print_usage "'--host' requires a host as parameter."
					else
						if [[ "$1" == --* ]]; then
							local _msg="A host should not start with '--'."
							_msg="$_msg Parameter for '--host' missing?"
							perror $_msg 
						fi
						declare -rg REMOTE_HOST="$1"
						shift
					fi
					continue
					;;
				--container)
					if [ -z "$1" ]; then 
						print_usage "'--container' requires a path as parameter."
					else
						if [[ "$1" == --* ]]; then
							local _msg="A path should not start with '--'."
							_msg="$_msg Parameter for '--container' missing?"
							perror $_msg 
						fi
						declare -rg CONTAINER_PATH="$1"
						shift
					fi
					continue
					;;
				--syncdir)
					if [ -z "$1" ]; then
						print_usage "'--syncdir' requires a path as parameter."
					else
						if [[ "$1" == --* ]]; then
							local _msg="A path should not start with '--'."
							_msg="$_msg Parameter for '--syncdir' missing?"
							perror $_msg 
						fi
						declare -rg RSYNC_TARGET="$1"
						shift
					fi
					continue
					;;
				--force)
					declare -rg FORCE=1
					;;
				*)
					print_usage "Unknown flag: $PARAM"
					;;
			esac
			continue
		fi
	done
	# done parsing the arguments, exit if no action given
	[ -z "$ACTION" ] && \
		print_usage "No action given"

	# now check for existance of variables
	# and use fallbacks when possible if they were not specified

	# REMOTE_HOST valid? always needed!
	[ -z "$REMOTE_HOST" ] && \
		perror "REMOTE_HOST not set. Use '--host'."
	check_host "$REMOTE_HOST" || \
		perror "'$REMOTE_HOST' is neither an IP nor a known hostname."
	# REMOTE_HOST valid - set build directory for the rest of the operations
	declare -rg BUILD_DIR="${ROOT_DIR}/clones/${REMOTE_HOST}"
	mkdir -p "${BUILD_DIR}" || perror "Could not create '${BUILD_DIR}'."

	# RSYNC_TARGET needs special care
	if [ -z "$RSYNC_TARGET" ]; then
		# none given - use fallbacks
		if [ "x$ACTION" == "xCLONE" ]; then
			# use default path when cloning, no need for CONTAINER_MNT
			pwarning "RSYNC_TARGET not set, using: '${BUILD_DIR}/stage4/'."
			declare -rg RSYNC_TARGET="${BUILD_DIR}/stage4"
		else
			# we always want CONTAINER_MNT here
			declare -rg CONTAINER_MNT="$(mktemp -d)"
			[ -z "${CONTAINER_MNT}" ] && \
				perror "Could not create temporary directory for mounting the container."
			add_cleanup rmdir "${CONTAINER_MNT}"
			# RSYNC_TARGET depends on the action at this point
			if [ "x$ACTION" == "xPACKAGE" ]; then
				# use default path when packaging
				declare -rg RSYNC_TARGET="${BUILD_DIR}/stage4"
			elif [ "x$ACTION" == "xUPDATE" -o "x$ACTION" == "xEXPORT" ]; then
				# for action update/export, we want to sync to the mounted container
				# so create a temporary directory for the mount point that we'll use later
				declare -rg RSYNC_TARGET="${CONTAINER_MNT}"
			fi
		fi
	fi

	# CONTAINER_PATH valid?
	if [ -z "$CONTAINER_PATH" ]; then
		# use default path: ${BUILD_DIR}/stage4.qcow2
		pwarning "CONTAINER_PATH not set. Using '${BUILD_DIR}/stage4.qcow2'."
		declare -rg CONTAINER_PATH="${BUILD_DIR}/stage4.qcow2"
	fi
	# so from now on REMOTE_HOST, RSYNC_TARGET, CONTAINER_PATH are set (and read-only).
}
process_action() {
	if [ "x$ACTION" == "xCLONE" ]; then
		clone_host || perror "Cloning stage4 failed with: $?"
	elif [ "x$ACTION" == "xPACKAGE" ]; then
		pack_clone || perror "Packing as QCoW2 failed with: $?"
	elif [ "x$ACTION" == "xEXPORT" ]; then
		export_host || perror "Exporting failed with: $?"
	elif [ "x$ACTION" == "xUPDATE" ]; then
		update_container || perror "Updating failed with: $?"
	else
		print_usage "No action given."
	fi
	return 0
}

# wrapper to package a cloned stage4 as a qcow2 container
#	- creates empty container at CONTAINER_PATH
#	- mounts it to RSYNC_TARGET
#	- copy RSYNC_TARGET
pack_clone() {
	create_container
	mount_container
	copy_to_container
}
# wrapper to update an existing container
#	- mounts it to RSYNC_TARGET
#	- clone host there
update_container() {
	mount_container
	clone_host
}
# wrapper to export a host directly to a container
#	- create en empty qcow2 container at CONTAINER_PATH
#	- mount it to RSYNC_TARGET
#	- clone host there
export_host() {
	create_container
	mount_container
	clone_host
}
# ------------------------------------------------------------------------------
#
#                          Stage4 related functions
#
# ------------------------------------------------------------------------------
#
# Helper to generate a stage4 export for a remote machine per rsync.
# Usage:
#			clone_host
#
# Note: this functions requires REMOTE_HOST and RSYNC_TARGET to be set.
clone_host() {
	# check if RSYNC_TARGET is valid
	if [ -d "${RSYNC_TARGET}" ]; then
		# does it have the '.stage4' flag? skip this check when exporting directly
		[ "x$ACTION" != "xEXPORT" ] && [ ! -e "${RSYNC_TARGET}/.stage4" ] && \
			perror "'${RSYNC_TARGET}' exists, but no '.stage4' flag found. Refusing to rsync there."
			#touch $RSYNC_TARGET/.stage4 && exit 0
	else
		# not a directory, create it and set the .stage4 flag
		mkdir -p "${RSYNC_TARGET}" || perror "Could not create '${RSYNC_TARGET}'."
	fi

	local EXCLUDE="$BUILD_DIR/exclude-stage4"
	local INCLUDE="$BUILD_DIR/include-stage4"

	pinfo "Building rsync include/exclude files for building stage4...."
	echo "## Exclude file for stage4 of $REMOTE_HOST" > "$EXCLUDE"
	echo "## Include file for stage4 of $REMOTE_HOST" > "$INCLUDE"
	for FILE in $(find "${ROOT_DIR}"/blacklists/*/ -type f); do
		echo "## From $FILE" >> "$EXCLUDE"
		echo "## From $FILE" >> "$INCLUDE"
		grep '^-' "$FILE" >> "$EXCLUDE"
		grep '^+' "$FILE" >> "$INCLUDE"
	done

	# prepare rsync's general options
	local RSYNC_OPTS="${DEFAULT_RSYNC_OPTS}"
	[ -z "${RSYNC_OPTS}" ] && \
		RSYNC_OPTS="
			--acls \
			--hard-links \
			--xattrs \
			--archive \
			--delete \
			--delete-excluded \
			--numeric-ids
		"
	# prepare rsync's remote shell options
	local RSYNC_RSH="$DEFAULT_RSYNC_RSH"
	[ -z "${RSYNC_RSH}" ] && RSYNC_RSH="ssh -c blowfish -oStrictHostKeyChecking=no"
	local RSYNC_SOURCE="root@$REMOTE_HOST:/"

	# if something goes wrong during rsync, we need to recreate the .stage4 flag
	add_cleanup touch ${RSYNC_TARGET}/.stage4
	# run rsync with the exclude/include lists created earlier
	cat "$INCLUDE" "$EXCLUDE" | \
		rsync ${RSYNC_OPTS} \
				--exclude-from=- \
            --rsh "${RSYNC_RSH}" \
				"${RSYNC_SOURCE}" \
				"${RSYNC_TARGET}"

	local -i rsync_ret=$?
	if [ "x$rsync_ret" != "x0" ]; then
		perror "rsync from '${RSYNC_SOURCE}' to '${RSYNC_TARGET}' failed."
	fi
	touch "${RSYNC_TARGET}/.stage4"
	# make sure everything gets flushed
	sync
	pinfo "Cloning '${REMOTE_HOST}' to '${RSYNC_TARGET}' succeeded."
}

# Helper to create the empty container at CONTAINER_PATH
create_container() {
	# CONTAINER_PATH valid?
	[ -d "${CONTAINER_PATH}" ] && perror "Path to container can not be a directory!"
	if [ -f "${CONTAINER_PATH}" ]; then
		if [ $FORCE -eq 0 ]; then
			perror "Container file already exists. Use '--force' to overwrite it."
		else
			# force removal
			rm -f "${CONTAINER_PATH}" || perror "Could not remove '${CONTAINER_PATH}'"
			pinfo "Removed old '${CONTAINER_PATH}'."
		fi
	fi

	# which size for the qcow2 container?
	local CONTAINER_SIZE="${DEFAULT_CONTAINER_SIZE}"
	[ -z "${CONTAINER_SIZE}" ] && CONTAINER_SIZE="20G"

	# so far so good
	pinfo "Creating qcow2-container '${CONTAINER_PATH}'"
	qemu-img create -f qcow2 "${CONTAINER_PATH}" "${CONTAINER_SIZE}" || \
		perror "qemu-img create failed with: $?"
	# now expose it as a loop device
	expose_container
	# filesystem for the qcow2 container?
	local CONTAINER_FILESYSTEM="${DEFAULT_CONTAINER_FILESYSTEM}"
	[ -z "${CONTAINER_FILESYSTEM}" ] && CONTAINER_FILESYSTEM="xfs"
	# check if we have that mkfs helper
	which "mkfs.${CONTAINER_FILESYSTEM}" || \
		perror "Could not find 'mkfs.${CONTAINER_FILESYSTEM}'. Install it and retry." 

	pinfo "Creating '${CONTAINER_FILESYSTEM}' filesystem on '${CONTAINER_PATH}'..."
	mkfs."${CONTAINER_FILESYSTEM}" "${LOOPED_NBD_DEV}" || perror "mkfs failed with: $?"
	return 0
}

# Helper exposing the container as a loop device
expose_container() {
	[ -z "${CONTAINER_PATH}" ] && \
		perror "Internal error - CONTAINER_PATH not set but should be! Check read_params()"
	
	# find usable nbd device
	declare -rg NBD_DEV="$(find_free_nbd)"
	[ -n "${NBD_DEV}" ] || perror "Could not find usable NBD device."
	[ -b "${NBD_DEV}" ] || perror "'${NBD_DEV}' is not a block device!"
	pinfo "Connecting '${CONTAINER_PATH}' to '${NBD_DEV}'"

	qemu-nbd -c "${NBD_DEV}" "${CONTAINER_PATH}" || \
		perror "qemu-nbd failed with: $?"
	add_cleanup disconnect_nbd
	# expose as a loop device
	declare -rg LOOPED_NBD_DEV="$(losetup --find)"
	losetup "${LOOPED_NBD_DEV}" "${NBD_DEV}" || \
		perror "Loop device setup for '${NBD_DEV}' failed with: $?"
	add_cleanup disconnect_loop
	return 0
}

# Helper to mount CONTAINER_PATH to CONTAINER_MNT through expose_container
mount_container() {
	[ -z "${CONTAINER_MNT}" ] && \
		perror "Internal error - CONTAINER_MNT not set but should be! Check read_params()"

	# connect container to a loop device first, if it wasnt already done
	[ -z "${LOOPED_NBD_DEV}" ] && expose_container
	# lets be safe...
	[ -z "${LOOPED_NBD_DEV}" ] && \
		perror "Internal error - LOOPED_NBD_DEV not set but should be! Check expose_container()"
	
	# now we got everything, mount it
	pinfo "Mounting '${LOOPED_NBD_DEV}' to '${CONTAINER_MNT}'..."
	mount "${LOOPED_NBD_DEV}" "${CONTAINER_MNT}" \
		|| perror "Mount failed with: $?"
	add_cleanup umount_container
	return 0
}

# helper to copy the content of RSYNC_TARGET to CONTAINER_MNT
copy_to_container() {
	[ -z "${RSYNC_TARGET}" ] && \
		perror "Internal error - RSYNC_TARGET not set but should be!"
	[ -z "${CONTAINER_MNT}" ] && \
		perror "Internal error - CONTAINER_MNT not set but should be!"
	# sanity checks
	is_dir_empty "$RSYNC_TARGET" && \
		perror "'$RSYNC_TARGET' appears to be empty. Did you clone?"
	# check for '.stage4' flag in the directory, indicating we cloned there
	if [ ! -e "${RSYNC_TARGET}/.stage4" ]; then
		perror "No '.stage4' flag found in '${RSYNC_TARGET}'." \
					"Was this cloned properly?"
	fi
	# copy files from the stage4 directory to the mounted qcow2-container
	pinfo "Copying '${RSYNC_TARGET}' to '${CONTAINER_MNT}'..."
	rsync -avAHX "${RSYNC_TARGET}"/ "${CONTAINER_MNT}"/ \
		|| perror "Rsync failed with: $?"
	#make sure everything is flushed
	sync && return 0
}


###############################################################################
#
#
#
###############################################################################
# helper to find an unused nbd device
# Usage:
#			find_free_nbd
# Echoes the name of the free device to stdout, empty string otherwise.
find_free_nbd() {
	local nbd_size=0
	for nbd_id in {0..15}; do
		[ -b "/dev/nbd${nbd_id}" ] || continue
		[ -r "/sys/block/nbd${nbd_id}/size" ] || continue
		nbd_size=$(cat /sys/block/nbd${nbd_id}/size)
		[ $nbd_size -eq 0 ] && echo "/dev/nbd${nbd_id}" && break
	done
	echo ""
}

# helper to validate an ip
# Usage:
#			valid_ip <ip>
# Returns 0 if valid, 1 otherwise.
valid_ip() {
    local  ip=$1
    local  stat=1

    if [[ $ip =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then
        OIFS=$IFS
        IFS='.'
        ip=($ip)
        IFS=$OIFS
        [[ ${ip[0]} -le 255 && ${ip[1]} -le 255 \
            && ${ip[2]} -le 255 && ${ip[3]} -le 255 ]]
        stat=$?
    fi
    return $stat
}

# helper to check whether a given host is valid
# Usage:
#			check_host <hostname|ip>
# Returns 0 if valid, 1 otherwise
check_host() {
	local HOST="$1"
	[ -z "$HOST" ] && return 1
	# check if its a valid IP or a valid hostname
	valid_ip "$HOST" && return 0
	host -W 2 "$HOST" && return 0
	# still here? then fail
	return 1
}

# helper to check if a dir is empty or not
# Usage:
#			dir_empty <dir>
# Returns 0 if empty, 1 otherwise
is_dir_empty() {
	[ $# -ne 1 ] && perror "$0 requires directory as paramter, none given."
	local _dir="$1"
	[ -d "$_dir" ] || return 1
	[ -n "$(ls -A $_dir)" ] && return 1 || return 0
}

# helper to ask user for confirmation
# Usage:
#			user_confirm
# Return 0 if confirmed, 1 otherwise
user_confirm() {
	[ $# -ne 1 ] && perror "$0 requires the question as first argument."
	pinfo "$1 [Y/n]"
	local _input
	read _input
	[ "x${_input}" == "x" -o "x${_input}" == "xy" ] && return 0 || return 1
}

###############################################################################
#
# CLEANUP FUNCTIONS
#
###############################################################################
# indexed array for cleanup commands
declare -ag cleanup_commands

# function to add a cleanup command to be executed when the program exits
add_cleanup() {
	# get current command count
	local -i count="${#cleanup_commands[*]}"
	cleanup_commands[$count]="$*"
}

# function trapped to EXIT, SIGINT, SIGTERM
# do the cleanup in FILO style
do_cleanup() {
	trap '' SIGINT SIGTERM EXIT	# from now on, ignore INT and TERM
	for i in $(seq $(( ${#cleanup_commands[*]} - 1 )) -1 0); do
		eval ${cleanup_commands[$i]}
	done
}

# Helper to umount + disconnect the container from all the devices
umount_container() {
	[ -z "${CONTAINER_MNT}" ] && \
		perror "CONTAINER_MNT not set - is it really mounted?"
	# sync?
	umount -l "${CONTAINER_MNT}" || \
		perror "Failed to umount '${CONTAINER_MNT}'."
#	rmdir "${CONTAINER_MNT}" || \
#		pwarning "Could not remove '${CONTAINER_MNT}'."
}

# Helper to disconnect from loop device
disconnect_loop() {
	[ -z "${LOOPED_NBD_DEV}" ] && \
		perror "Container not connected to a loop device?"
	losetup -d "${LOOPED_NBD_DEV}" ||\
		perror "Could not disconnect loop device '${LOOPED_NBD_DEV}'."
}
# Helper to disconnect from nbd device
disconnect_nbd() {
	[ -z "${NBD_DEV}" ] && \
		perror "Container does not seem to be connected to a NBD device?"
	qemu-nbd -d "${NBD_DEV}" || \
		perror "Could not disconnect '${CONTAINER_PATH}' from '${NBD_DEV}'."
}