path: root/packager/openslx.functions

# ------------------------------------------------------------------------------
#
#                          Functions for OpenSLX-NG
#
# ------------------------------------------------------------------------------
#
# check if we have our environment variables to check
# if we actually got sourced by the main script
if [ -z "${SELF_PID}" -o  -z "${ROOT_DIR}" ]; then
	# not using perror, since we probably don't have it.
	echo "Neither SELF_PID nor ROOT_DIR is set. Was this included by OpenSLX-NG?"
	exit 1
fi

# ------------------------------------------------------------------------------
#
#                          General helper functions
#
# ------------------------------------------------------------------------------
banner () {
	echo -e "\033[38;5;196m\t"'                                    .__          '
	echo -e "\033[38;5;196m\t"'  ____ ______   ____   ____   _____|  | ___  ___ '
	echo -e "\033[38;5;202m\t"' /  _ \\\\____ \\_/ __ \\ /    \\ /  ___/  | \\  \\/  / '
	echo -e "\033[38;5;208m\t"'(  <_> )  |_> >  ___/|   |  \\\\___ \\|  |__>    <  '
	echo -e "\033[38;5;214m\t"' \\____/|   __/ \\___  >___|  /____  >____/__/\\_ \\ '
	echo -e "\033[38;5;220m\t"'       |__|        \\/     \\/     \\/           \\/ '
	echo -e "\033[38;5;220m\t"
	echo -e "\033[38;5;226m\t                     ** OpenSLX Project // 2015 **"
	echo -e "\033[38;5;232m\t                        http://lab.openslx.org/"
	echo -e "\033[00m"
}

pinfo() {
	echo -e "\033[38;5;10m[info]\033[0m $@"
}

pwarning() {
	echo -e "\033[38;5;11m[warning]\033[0m $@"
}

perror() {
	echo -e "\033[38;5;9m[error]\033[0m $@"
	kill "$SELF_PID"
	exit 1
}

print_usage() {
	pinfo "USAGE:"
	pinfo "$ARG0 <action> <actions_params>"
	pinfo "\t\tActions: '--clone', '--package'"
	pinfo ""
	pinfo "CLONING:"
	pinfo "$ARG0 --clone --host <host> [--syncdir <path>]"
	pinfo "\t\tIf not specified, --syncdir = './builds/<host>/stage4'"
	pinfo ""
	pinfo "PACKAGING:"
	pinfo "$ARG0 --package --syncdir <path> --container <path>"
	pinfo "$ARG0 --package --host <host> --container <path>"
	pinfo "\t\tIf <host> is specified, --syncdir = './builds/<host>/stage4'"
	pinfo "$ARG0 --package --host <host> --syncdir <path> --container <path>"
	pinfo "\t\tIf both are specified, --syncdir is used."
	kill "$SELF_PID"
	exit 1
}
#
# helper to parse the command line arguments and fill the environment
# with the parameters given. Note that checks for validity happens
# in the respective functions, we only parse here.
read_params() {
	# initialize global variables
	declare -g FORCE=0
	unset ACTION REMOTE_HOST CONTAINER_PATH RSYNC_TARGET

	# handle rest of arguments
	while [ "$#" -gt "0" ]; do
		local PARAM="$1"
		shift

		# options to current target
		if [[ "$PARAM" == --* ]]; then
			case "$PARAM" in
				--clone)
					declare -rg ACTION="CLONE"
					;;
				--package)
					declare -rg ACTION="PACKAGE"
					;;
				--host)
					if [ -z "$1" ]; then 
						pwarning "'--host' requires a host as parameter."
						print_usage && exit 1
					else
						if [[ "$1" == --* ]]; then
							local _msg="A host should not start with '--'."
							_msg="$_msg Parameter for '--host' missing?"
							perror $_msg 
						fi
						declare -rg REMOTE_HOST="$1"
						shift
					fi
					continue
					;;
				--container)
					if [ -z "$1" ]; then 
						pwarning "'--container' requires a path as parameter."
						print_usage && exit 1
					else
						if [[ "$1" == --* ]]; then
							local _msg="A path should not start with '--'."
							_msg="$_msg Parameter for '--container' missing?"
							perror $_msg 
						fi
						declare -rg CONTAINER_PATH="$1"
						shift
					fi
					continue
					;;
				--syncdir)
					if [ -z "$1" ]; then
						pwarning "'--syncdir' requires a path as parameter."
						print_usage && exit 1
					else
						if [[ "$1" == --* ]]; then
							local _msg="A path should not start with '--'."
							_msg="$_msg Parameter for '--syncdir' missing?"
							perror $_msg 
						fi
						declare -rg RSYNC_TARGET="$1"
						shift
					fi
					continue
					;;
				--force)
					declare -rg FORCE=1
					;;
				*)
					pwarning "Unknown flag: $PARAM"
					print_usage && exit 1
					;;
			esac
			continue
		fi
	done
}

# helper function trapped on SIGTERM/SIGINT
# Usage: do not use as is
cleanexit() {
	trap '' SIGINT SIGTERM	# from now on, ignore INT and TERM
	pwarning "SIGINT/SIGTERM triggered - cleaning up ..."
	[ -z "${_STATE}" ] && perror "'_STATE' not set, this is bad."

	case "${_STATE}" in
		SYNC_DONE|QCOW_DONE)
			# we are happy
			pwarning "SIGINT/SIGTERM received, but everything seems fine. Check it!"
			exit 0
			;;
		BLACKLISTING)
			# creation of blacklists failed
			# TODO do what?
			;;
		SYNCING)
			# error during rsync, create the .stage4 file again
			[ -z "${RSYNC_TARGET}" ] && \
				perror "RSYNC_TARGET not set, this should not happen."
			if [ ! -e "${RSYNC_TARGET}/.stage4" ]; then
				pwarning "'.stage4' flag was lost during rsync, restoring it."
				touch "${RSYNC_TARGET}/.stage4"
			fi
			;;
		QCOW_CREATING)
			# qemu-img failed. Just remove the container if its there
			if [ -n "${CONTAINER_PATH}" -a -e "${CONTAINER_PATH}" ]; then
				rm -f "${CONTAINER_PATH}" || \
					pwarning "Could not remove '${CONTAINER_PATH}'."
			fi
			;;
		QCOW_NBD_CONNECTING)
			# qemu-nbd failed
			if [ -n "${NBD_DEV}" ]; then
				qemu-nbd -d "${NBD_DEV}" && \
					pwarning "Could not disconnect '${NBD_DEV}'."
			fi
			;;
		QCOW_FSING)
			# mkfs failed, disconnect and remove container
			if [ -n "${NBD_DEV}" ]; then
				qemu-nbd -d "${NBD_DEV}" && \
					pwarning "Could not disconnect '${NBD_DEV}'."
			fi
			if [ -n "${CONTAINER_PATH}" -a -e "${CONTAINER_PATH}" ]; then
				rm -f "${CONTAINER_PATH}" || \
					pwarning "Could not remove '${CONTAINER_PATH}'."
			fi
			;;
		QCOW_MOUNTING)
			# mounting failed:
			# umount, disconnect and remove container and mount point
			if [ -n "${NBD_MNT}" ]; then
				umount "${NBD_MNT}" || pwarning "Could not umount '${NBD_MNT}'."
				rmdir "${NBD_MNT}" || pwarning "Could not rmdir '${NBD_MNT}'."
			fi
			if [ -n "${NBD_DEV}" ]; then
				qemu-nbd -d "${NBD_DEV}" && \
					perror "Could not disconnect '${NBD_DEV}'."
			fi
			if [ -n "${CONTAINER_PATH}" -a -e "${CONTAINER_PATH}" ]; then
				rm -f "${CONTAINER_PATH}" || \
					perror "Could not remove '${CONTAINER_PATH}'."
			fi
			;;
		QCOW_COPYING)
			# rare case, should not happen
			;;
		QCOW_CLEANING)
			# should also not happen
			;;
		*)
			pwarning "Unknown state: ${_STATE}"
	esac
	# still here? then we ran into some error
	exit 1
}



# helper to validate an ip
# Usage:
#			valid_ip <ip>
# Returns 0 if valid, 1 otherwise.
valid_ip() {
    local  ip=$1
    local  stat=1

    if [[ $ip =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then
        OIFS=$IFS
        IFS='.'
        ip=($ip)
        IFS=$OIFS
        [[ ${ip[0]} -le 255 && ${ip[1]} -le 255 \
            && ${ip[2]} -le 255 && ${ip[3]} -le 255 ]]
        stat=$?
    fi
    return $stat
}

# helper to check whether a given host is valid
# Usage:
#			check_host <hostname|ip>
# Returns 0 if valid, 1 otherwise
check_host() {
	local HOST="$1"
	[ -z "$HOST" ] && return 1
	# check if its a valid IP or a valid hostname
	valid_ip "$HOST" && return 0
	host -W 2 "$HOST" && return 0
	# still here? then fail
	return 1
}

# helper to check if a dir is empty or not
# Usage:
#			dir_empty <dir>
# Returns 0 if empty, 1 otherwise
is_dir_empty() {
	[ $# -ne 1 ] && perror "$0 requires directory as paramter, none given."
	local _dir="$1"
	[ -d "$_dir" ] || return 1
	[ -n "$(ls -A $_dir)" ] && return 1 || return 0
}

# helper to ask user for confirmation
# Usage:
#			user_confirm
# Return 0 if confirmed, 1 otherwise
user_confirm() {
	[ $# -ne 1 ] && perror "$0 requires the question as first argument."
	pinfo "$1 [Y/n]"
	local _input
	read _input
	[ "x${_input}" == "x" -o "x${_input}" == "xy" ] && return 0 || return 1
}

# ------------------------------------------------------------------------------
#
#                          Stage4 related functions
#
# ------------------------------------------------------------------------------
#
# Helper to generate a stage4 export for a remote machine per rsync.
# Usage:
#			clone_stage4
#
# Note: this functions requires REMOTE_HOST and RSYNC_TARGET to be set.
clone_stage4() {
	# REMOTE_HOST valid?
	[ -z "$REMOTE_HOST" ] && pwarning "REMOTE_HOST not set. Use '--host'." && print_usage
	if ! check_host "$REMOTE_HOST"; then
		# invalid, abort
		pwarning "'$REMOTE_HOST' is neither an IP nor a known hostname."
		print_usage
	fi
	# REMOTE_HOST is valid, use it as the base for our build files
	# set BUILD_DIR relative to the ROOT_DIR for the REMOTE_HOST
	declare -rg BUILD_DIR="${ROOT_DIR}/builds/$REMOTE_HOST"
	mkdir -p "${BUILD_DIR}"

	# RSYNC_TARGET set?
	if [ -z "$RSYNC_TARGET" ]; then
		pwarning "RSYNC_TARGET not set. Assuming local mode."
		pinfo "Using '${BUILD_DIR}/stage4'"
		declare -rg RSYNC_TARGET="${BUILD_DIR}/stage4"
	fi

	# check if RSYNC_TARGET is valid
	if [ -d "${RSYNC_TARGET}" ]; then
		# does it have the '.stage4' flag?
		[ ! -e "${RSYNC_TARGET}/.stage4" ] && \
			perror "'${RSYNC_TARGET}' exists, but no '.stage4' flag found." \
					 "Refusing to rsync there."
	else
		# not a directory, create it and set the .stage4 flag
		mkdir -p "${RSYNC_TARGET}"
		touch "${RSYNC_TARGET}/.stage4"
	fi

	# mark state
	_STATE='BLACKLISTING'
	local EXCLUDE="$BUILD_DIR/exclude-stage4"
	local INCLUDE="$BUILD_DIR/include-stage4"

	pinfo "Building rsync include/exclude files for building stage4...."
	echo "## Exclude file for stage4 of $REMOTE_HOST" > "$EXCLUDE"
	echo "## Include file for stage4 of $REMOTE_HOST" > "$INCLUDE"
	for FILE in $(find "${ROOT_DIR}"/blacklists/*/ -type f); do
		echo "## From $FILE" >> "$EXCLUDE"
		echo "## From $FILE" >> "$INCLUDE"
		grep '^-' "$FILE" >> "$EXCLUDE"
		grep '^+' "$FILE" >> "$INCLUDE"
	done
	pinfo "Done."

	# prepare rsync's options
	if [ -z "$DEFAULT_RSYNC_OPTS" ]; then
		local RSYNC_OPTS="-e ssh -c arcfour -oStrictHostKeyChecking=no"
	else
		local RSYNC_OPTS="$DEFAULT_RSYNC_OPTS"
	fi
	local RSYNC_SOURCE="root@$REMOTE_HOST:/"

	_STATE='SYNCING'
	# run rsync with the exclude/include lists created earlier
	cat "$INCLUDE" "$EXCLUDE" | \
		rsync --verbose \
				--acls \
				--hard-links \
				--xattrs \
				--archive \
				--delete \
				--delete-excluded \
				--numeric-ids \
				--exclude-from=- \
				"${RSYNC_OPTS}" \
				"${RSYNC_SOURCE}" \
				"${RSYNC_TARGET}" \
		|| perror "rsync from '${RSYNC_SOURCE}' to '${RSYNC_TARGET}' failed."
	## TODO real exit code handling
	pinfo "Cloning '${REMOTE_HOST}' to '${RSYNC_TARGET}' succeeded."
	_STATE='SYNC_DONE'
	touch "${RSYNC_TARGET}/.stage4"
	return 0
}
# helper to build a qcow2 container from a stage4 sync directory
# Usage:
#			pack_qcow2
#
# Note that this requires CONTAINER_PATH to be set.
# RSYNC_TARGET is either the path given through the option '--syncdir'
# or the standard local path '$ROOT_DIR/builds/$REMOTE_HOST/stage4'
# is assumed to be our rsync destination directory.
pack_qcow2() {
	# CONTAINER_PATH valid?
	[ -z "$CONTAINER_PATH" ] && \
		pwarning "CONTAINER_PATH not set. Use '--container'." && print_usage
	[ -d "$CONTAINER_PATH" ] && perror "Path to container can not be a directory!"
	if [ -f "$CONTAINER_PATH" ]; then
		if [ $FORCE -eq 0 ]; then
			perror "Container file already exists. Use '--force' to overwrite it."
		else
			# force removal
			rm -f "$CONTAINER_PATH" || perror "Could not remove '$CONTAINER_PATH'"
			pinfo "Removed old '$CONTAINER_PATH'."
		fi
	fi

	# RSYNC_TARGET valid?
	if [ -z "$RSYNC_TARGET" ]; then
		# if not RSYNC_TARGET was specified, we must have REMOTE_HOST
		# or we do not know which stage4 we are supposed to package
		[ -z "$REMOTE_HOST" ] && \
			pwarning "Need either '--syncdir' or '--host'!" && \
			print_usage
		check_host "$REMOTE_HOST" || perror "Given host invalid."
		pwarning "RSYNC_TARGET not set. Assuming local mode."
		local RSYNC_TARGET_CANDIDATE="${ROOT_DIR}/builds/${REMOTE_HOST}/stage4"
		if [ ! -d "$RSYNC_TARGET_CANDIDATE" ]; then
			pwarning "Local stage4 sync not found at '${RSYNC_TARGET_CANDIDATE}'"
			pwarning "Did you sync with '--syncdir' set? Then use that :)"
			perror "Stage4 to package not found."
		fi
		is_dir_empty "$RSYNC_TARGET_CANDIDATE" && \
			perror "Appears to be empty, did you clone?"
		pinfo "Found '$RSYNC_TARGET_CANDIDATE', using it."
		declare -rg RSYNC_TARGET="$RSYNC_TARGET_CANDIDATE"
	fi

	# more sanity checks
	[ ! -d "$RSYNC_TARGET" ] && perror "'$RSYNC_TARGET' not a directory!"
	is_dir_empty "$RSYNC_TARGET" && \
		perror "'$RSYNC_TARGET' appears to be empty. Did you clone?"
	# the ultimative check
	if [ ! -e "${RSYNC_TARGET}/.stage4" ]; then
		perror "No '.stage4' flag found in '${RSYNC_TARGET}'." \
					"Was this cloned properly?"
	fi

	# which size for the qcow2 container?
	if [ -z "$DEFAULT_QCOW_SIZE" ]; then
		local QCOW_SIZE="25G"
	else
		local QCOW_SIZE="$DEFAULT_QCOW_SIZE"
	fi
	# so far so good
	pinfo "Creating empty qcow2-container ..."
	_STATE='QCOW_CREATING'
	qemu-img create -f qcow2 "${CONTAINER_PATH}" "${QCOW_SIZE}" \
		|| perror "qemu-img create failed with: $?"
	pinfo "Done."

	# find usable nbd device
	pinfo "Looking for usable nbd device..."
	local NBD_DEV="$(find_free_nbd)"
	[ -z "${NBD_DEV}" ] && perror "Could not find usable NBD device."
	[ -b "${NBD_DEV}" ] || perror "'${NBD_DEV}' is not a block device!"
	pinfo "Exporting '${CONTAINER_PATH}' using '${NBD_DEV}'..."
	_STATE='QCOW_NBD_CONNECTING'
	qemu-nbd -c "${NBD_DEV}" "${CONTAINER_PATH}" || \
		perror "qemu-nbd failed with: $?"
	pinfo "Done."

	# expose as a loop device
	local NBD_LOOPED_DEV="$(losetup --find)"
	losetup "${NBD_LOOPED_DEV}" "${NBD_DEV}"

	# which filesystem for the qcow2 container?
	if [ -z "$DEFAULT_QCOW_FS" ]; then
		local QCOW_FS="ext4"
	else
		# check if we have mkfs helper
		which "mkfs.$DEFAULT_QCOW_FS" &>/dev/null || \
			perror "Could not find 'mkfs.$DEFAULT_QCOW_FS'." 
		local QCOW_FS="$DEFAULT_QCOW_FS"
	fi
	pinfo "Creating '${QCOW_FS}' filesystem on '${CONTAINER_PATH}'..."
	_STATE='QCOW_FSING'
	mkfs."${QCOW_FS}" "${NBD_LOOPED_DEV}" || perror "mkfs failed with: $?"
	pinfo "Done."

	
	# prepare NBD mount directory and check state to be safe
	local NBD_MNT="$(mktemp -d)"
	[ ! -d "${NBD_MNT}" ] && \
		perror "Making temporary dir for mounting '${NBD_DEV}' failed."
	is_dir_empty ${NBD_MNT} || \
		perror "'${NBD_MNT}' not empty. Refusing to mount '${NBD_DEV}' to it."

	pinfo "Mounting '${NBD_DEV}' to '${NBD_MNT}'..."
	_STATE='QCOW_MOUNTING'
	mount "${NBD_LOOPED_DEV}" "${NBD_MNT}" || perror "Mount failed with: $?"
	pinfo "Done."

	# copy files from the stage4 directory to the mounted qcow2-container
	pinfo "Copying '${RSYNC_TARGET}' to '${NBD_MNT}'..."
	_STATE='QCOW_COPYING'
	rsync -avAHX "${RSYNC_TARGET}"/ "${NBD_MNT}"/ || perror "Copying failed with: $?"
	pinfo "Done."

	pinfo "Cleaning up..."
	_STATE='QCOW_CLEANING'
	umount "${NBD_MNT}" || pwarning "Could not unmount '${NBD_MNT}'."
	rmdir "${NBD_MNT}" || pwarning "Could not remove '${NBD_MNT}'."
	losetup -d "${NBD_LOOPED_DEV}" || pwarning "Could not disconnect '${NBD_LOOPED_DEV}'."
	qemu-nbd -d "${NBD_DEV}" || pwarning "Could not disconnect '${NBD_DEV}'."
	_STATE='QCOW_DONE'
	pinfo "Exporting '${RSYNC_TARGET}' to '${CONTAINER_PATH}' completed."
}

# helper to find an unused nbd device
# Usage:
#			find_free_nbd
# Echoes the name of the free device to stdout, empty string otherwise.
find_free_nbd() {
	local nbd_size=0
	for nbd_id in {0..15}; do
		[ -b "/dev/nbd${nbd_id}" ] || continue
		[ -r "/sys/block/nbd${nbd_id}/size" ] || continue
		nbd_size=$(cat /sys/block/nbd${nbd_id}/size)
		[ $nbd_size -eq 0 ] && echo "/dev/nbd${nbd_id}" && break
	done
	echo ""
}