# ------------------------------------------------------------------------------
#
# Functions for OpenSLX-NG
#
# ------------------------------------------------------------------------------
#
# check if we have our environment variables to check
# if we actually got sourced by the main script
if [ -z "${SELF_PID}" -o -z "${ROOT_DIR}" ]; then
# not using perror, since we probably don't have it.
echo "Neither SELF_PID nor ROOT_DIR is set. Was this included by OpenSLX-NG?"
exit 1
fi
# ------------------------------------------------------------------------------
#
# General helper functions
#
# ------------------------------------------------------------------------------
banner () {
echo -e "\033[38;5;196m\t"' .__ '
echo -e "\033[38;5;196m\t"' ____ ______ ____ ____ _____| | ___ ___ '
echo -e "\033[38;5;202m\t"' / _ \\\\____ \\_/ __ \\ / \\ / ___/ | \\ \\/ / '
echo -e "\033[38;5;208m\t"'( <_> ) |_> > ___/| | \\\\___ \\| |__> < '
echo -e "\033[38;5;214m\t"' \\____/| __/ \\___ >___| /____ >____/__/\\_ \\ '
echo -e "\033[38;5;220m\t"' |__| \\/ \\/ \\/ \\/ '
echo -e "\033[38;5;220m\t"
echo -e "\033[38;5;226m\t ** OpenSLX Project // 2015 **"
echo -e "\033[38;5;232m\t http://lab.openslx.org/"
echo -e "\033[00m"
}
pinfo() {
echo -e "\033[38;5;10m[info]\033[0m $@"
}
pwarning() {
echo -e "\033[38;5;11m[warning]\033[0m $@"
}
perror() {
echo -e "\033[38;5;9m[error]\033[0m $@"
kill "$SELF_PID"
exit 1
}
print_usage() {
pinfo "USAGE:"
pinfo "$ARG0 <action> <actions_params>"
pinfo "\t\tActions: '--clone', '--package'"
pinfo ""
pinfo "CLONING:"
pinfo "$ARG0 --clone --host <host> [--syncdir <path>]"
pinfo "\t\tIf not specified, --syncdir = './builds/<host>/stage4'"
pinfo ""
pinfo "PACKAGING:"
pinfo "$ARG0 --package --syncdir <path> --container <path>"
pinfo "$ARG0 --package --host <host> --container <path>"
pinfo "\t\tIf <host> is specified, --syncdir = './builds/<host>/stage4'"
pinfo "$ARG0 --package --host <host> --syncdir <path> --container <path>"
pinfo "\t\tIf both are specified, --syncdir is used."
kill "$SELF_PID"
exit 1
}
#
# helper to parse the command line arguments and fill the environment
# with the parameters given. Note that checks for validity happens
# in the respective functions, we only parse here.
read_params() {
# initialize global variables
declare -g FORCE=0
unset ACTION REMOTE_HOST CONTAINER_PATH RSYNC_TARGET
# handle rest of arguments
while [ "$#" -gt "0" ]; do
local PARAM="$1"
shift
# options to current target
if [[ "$PARAM" == --* ]]; then
case "$PARAM" in
--clone)
declare -rg ACTION="CLONE"
;;
--package)
declare -rg ACTION="PACKAGE"
;;
--host)
if [ -z "$1" ]; then
pwarning "'--host' requires a host as parameter."
print_usage && exit 1
else
if [[ "$1" == --* ]]; then
local _msg="A host should not start with '--'."
_msg="$_msg Parameter for '--host' missing?"
perror $_msg
fi
declare -rg REMOTE_HOST="$1"
shift
fi
continue
;;
--container)
if [ -z "$1" ]; then
pwarning "'--container' requires a path as parameter."
print_usage && exit 1
else
if [[ "$1" == --* ]]; then
local _msg="A path should not start with '--'."
_msg="$_msg Parameter for '--container' missing?"
perror $_msg
fi
declare -rg CONTAINER_PATH="$1"
shift
fi
continue
;;
--syncdir)
if [ -z "$1" ]; then
pwarning "'--syncdir' requires a path as parameter."
print_usage && exit 1
else
if [[ "$1" == --* ]]; then
local _msg="A path should not start with '--'."
_msg="$_msg Parameter for '--syncdir' missing?"
perror $_msg
fi
declare -rg RSYNC_TARGET="$1"
shift
fi
continue
;;
--force)
declare -rg FORCE=1
;;
*)
pwarning "Unknown flag: $PARAM"
print_usage && exit 1
;;
esac
continue
fi
done
}
# helper function trapped on SIGTERM/SIGINT
# Usage: do not use as is
cleanexit() {
trap '' SIGINT SIGTERM # from now on, ignore INT and TERM
pwarning "SIGINT/SIGTERM triggered - cleaning up ..."
[ -z "${_STATE}" ] && perror "'_STATE' not set, this is bad."
case "${_STATE}" in
SYNC_DONE|QCOW_DONE)
# we are happy
pwarning "SIGINT/SIGTERM received, but everything seems fine. Check it!"
exit 0
;;
BLACKLISTING)
# creation of blacklists failed
# TODO do what?
;;
SYNCING)
# error during rsync, create the .stage4 file again
[ -z "${RSYNC_TARGET}" ] && \
perror "RSYNC_TARGET not set, this should not happen."
if [ ! -e "${RSYNC_TARGET}/.stage4" ]; then
pwarning "'.stage4' flag was lost during rsync, restoring it."
touch "${RSYNC_TARGET}/.stage4"
fi
;;
QCOW_CREATING)
# qemu-img failed. Just remove the container if its there
if [ -n "${CONTAINER_PATH}" -a -e "${CONTAINER_PATH}" ]; then
rm -f "${CONTAINER_PATH}" || \
pwarning "Could not remove '${CONTAINER_PATH}'."
fi
;;
QCOW_NBD_CONNECTING)
# qemu-nbd failed
if [ -n "${NBD_DEV}" ]; then
qemu-nbd -d "${NBD_DEV}" && \
pwarning "Could not disconnect '${NBD_DEV}'."
fi
;;
QCOW_FSING)
# mkfs failed, disconnect and remove container
if [ -n "${NBD_DEV}" ]; then
qemu-nbd -d "${NBD_DEV}" && \
pwarning "Could not disconnect '${NBD_DEV}'."
fi
if [ -n "${CONTAINER_PATH}" -a -e "${CONTAINER_PATH}" ]; then
rm -f "${CONTAINER_PATH}" || \
pwarning "Could not remove '${CONTAINER_PATH}'."
fi
;;
QCOW_MOUNTING)
# mounting failed:
# umount, disconnect and remove container and mount point
if [ -n "${NBD_MNT}" ]; then
umount "${NBD_MNT}" || pwarning "Could not umount '${NBD_MNT}'."
rmdir "${NBD_MNT}" || pwarning "Could not rmdir '${NBD_MNT}'."
fi
if [ -n "${NBD_DEV}" ]; then
qemu-nbd -d "${NBD_DEV}" && \
perror "Could not disconnect '${NBD_DEV}'."
fi
if [ -n "${CONTAINER_PATH}" -a -e "${CONTAINER_PATH}" ]; then
rm -f "${CONTAINER_PATH}" || \
perror "Could not remove '${CONTAINER_PATH}'."
fi
;;
QCOW_COPYING)
# rare case, should not happen
;;
QCOW_CLEANING)
# should also not happen
;;
*)
pwarning "Unknown state: ${_STATE}"
esac
# still here? then we ran into some error
exit 1
}
# helper to validate an ip
# Usage:
# valid_ip <ip>
# Returns 0 if valid, 1 otherwise.
valid_ip() {
local ip=$1
local stat=1
if [[ $ip =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then
OIFS=$IFS
IFS='.'
ip=($ip)
IFS=$OIFS
[[ ${ip[0]} -le 255 && ${ip[1]} -le 255 \
&& ${ip[2]} -le 255 && ${ip[3]} -le 255 ]]
stat=$?
fi
return $stat
}
# helper to check whether a given host is valid
# Usage:
# check_host <hostname|ip>
# Returns 0 if valid, 1 otherwise
check_host() {
local HOST="$1"
[ -z "$HOST" ] && return 1
# check if its a valid IP or a valid hostname
valid_ip "$HOST" && return 0
host -W 2 "$HOST" && return 0
# still here? then fail
return 1
}
# helper to check if a dir is empty or not
# Usage:
# dir_empty <dir>
# Returns 0 if empty, 1 otherwise
is_dir_empty() {
[ $# -ne 1 ] && perror "$0 requires directory as paramter, none given."
local _dir="$1"
[ -d "$_dir" ] || return 1
[ -n "$(ls -A $_dir)" ] && return 1 || return 0
}
# helper to ask user for confirmation
# Usage:
# user_confirm
# Return 0 if confirmed, 1 otherwise
user_confirm() {
[ $# -ne 1 ] && perror "$0 requires the question as first argument."
pinfo "$1 [Y/n]"
local _input
read _input
[ "x${_input}" == "x" -o "x${_input}" == "xy" ] && return 0 || return 1
}
# ------------------------------------------------------------------------------
#
# Stage4 related functions
#
# ------------------------------------------------------------------------------
#
# Helper to generate a stage4 export for a remote machine per rsync.
# Usage:
# clone_stage4
#
# Note: this functions requires REMOTE_HOST and RSYNC_TARGET to be set.
clone_stage4() {
# REMOTE_HOST valid?
[ -z "$REMOTE_HOST" ] && pwarning "REMOTE_HOST not set. Use '--host'." && print_usage
if ! check_host "$REMOTE_HOST"; then
# invalid, abort
pwarning "'$REMOTE_HOST' is neither an IP nor a known hostname."
print_usage
fi
# REMOTE_HOST is valid, use it as the base for our build files
# set BUILD_DIR relative to the ROOT_DIR for the REMOTE_HOST
declare -rg BUILD_DIR="${ROOT_DIR}/builds/$REMOTE_HOST"
mkdir -p "${BUILD_DIR}"
# RSYNC_TARGET set?
if [ -z "$RSYNC_TARGET" ]; then
pwarning "RSYNC_TARGET not set. Assuming local mode."
pinfo "Using '${BUILD_DIR}/stage4'"
declare -rg RSYNC_TARGET="${BUILD_DIR}/stage4"
fi
# check if RSYNC_TARGET is valid
if [ -d "${RSYNC_TARGET}" ]; then
# does it have the '.stage4' flag?
[ ! -e "${RSYNC_TARGET}/.stage4" ] && \
perror "'${RSYNC_TARGET}' exists, but no '.stage4' flag found." \
"Refusing to rsync there."
else
# not a directory, create it and set the .stage4 flag
mkdir -p "${RSYNC_TARGET}"
touch "${RSYNC_TARGET}/.stage4"
fi
# mark state
_STATE='BLACKLISTING'
local EXCLUDE="$BUILD_DIR/exclude-stage4"
local INCLUDE="$BUILD_DIR/include-stage4"
pinfo "Building rsync include/exclude files for building stage4...."
echo "## Exclude file for stage4 of $REMOTE_HOST" > "$EXCLUDE"
echo "## Include file for stage4 of $REMOTE_HOST" > "$INCLUDE"
for FILE in $(find "${ROOT_DIR}"/blacklists/*/ -type f); do
echo "## From $FILE" >> "$EXCLUDE"
echo "## From $FILE" >> "$INCLUDE"
grep '^-' "$FILE" >> "$EXCLUDE"
grep '^+' "$FILE" >> "$INCLUDE"
done
pinfo "Done."
# prepare rsync's options
if [ -z "$DEFAULT_RSYNC_OPTS" ]; then
local RSYNC_OPTS="-e ssh -c arcfour -oStrictHostKeyChecking=no"
else
local RSYNC_OPTS="$DEFAULT_RSYNC_OPTS"
fi
local RSYNC_SOURCE="root@$REMOTE_HOST:/"
_STATE='SYNCING'
# run rsync with the exclude/include lists created earlier
cat "$INCLUDE" "$EXCLUDE" | \
rsync --verbose \
--archive \
--delete \
--delete-excluded \
--numeric-ids \
--exclude-from=- \
"${RSYNC_OPTS}" \
"${RSYNC_SOURCE}" \
"${RSYNC_TARGET}" \
|| perror "rsync from '${RSYNC_SOURCE}' to '${RSYNC_TARGET}' failed."
## TODO real exit code handling
pinfo "Cloning '${REMOTE_HOST}' to '${RSYNC_TARGET}' succeeded."
_STATE='SYNC_DONE'
touch "${RSYNC_TARGET}/.stage4"
return 0
}
# helper to build a qcow2 container from a stage4 sync directory
# Usage:
# pack_qcow2
#
# Note that this requires CONTAINER_PATH to be set.
# RSYNC_TARGET is either the path given through the option '--syncdir'
# or the standard local path '$ROOT_DIR/builds/$REMOTE_HOST/stage4'
# is assumed to be our rsync destination directory.
pack_qcow2() {
# CONTAINER_PATH valid?
[ -z "$CONTAINER_PATH" ] && \
pwarning "CONTAINER_PATH not set. Use '--container'." && print_usage
[ -d "$CONTAINER_PATH" ] && perror "Path to container can not be a directory!"
if [ -f "$CONTAINER_PATH" ]; then
if [ $FORCE -eq 0 ]; then
perror "Container file already exists. Use '--force' to overwrite it."
else
# force removal
rm -f "$CONTAINER_PATH" || perror "Could not remove '$CONTAINER_PATH'"
pinfo "Removed old '$CONTAINER_PATH'."
fi
fi
# RSYNC_TARGET valid?
if [ -z "$RSYNC_TARGET" ]; then
# if not RSYNC_TARGET was specified, we must have REMOTE_HOST
# or we do not know which stage4 we are supposed to package
[ -z "$REMOTE_HOST" ] && \
pwarning "Need either '--syncdir' or '--host'!" && \
print_usage
check_host "$REMOTE_HOST" || perror "Given host invalid."
pwarning "RSYNC_TARGET not set. Assuming local mode."
local RSYNC_TARGET_CANDIDATE="${ROOT_DIR}/builds/${REMOTE_HOST}/stage4"
if [ ! -d "$RSYNC_TARGET_CANDIDATE" ]; then
pwarning "Local stage4 sync not found at '${RSYNC_TARGET_CANDIDATE}'"
pwarning "Did you sync with '--syncdir' set? Then use that :)"
perror "Stage4 to package not found."
fi
is_dir_empty "$RSYNC_TARGET_CANDIDATE" && \
perror "Appears to be empty, did you clone?"
pinfo "Found '$RSYNC_TARGET_CANDIDATE', using it."
declare -rg RSYNC_TARGET="$RSYNC_TARGET_CANDIDATE"
fi
# more sanity checks
[ ! -d "$RSYNC_TARGET" ] && perror "'$RSYNC_TARGET' not a directory!"
is_dir_empty "$RSYNC_TARGET" && \
perror "'$RSYNC_TARGET' appears to be empty. Did you clone?"
# the ultimative check
if [ ! -e "${RSYNC_TARGET}/.stage4" ]; then
perror "No '.stage4' flag found in '${RSYNC_TARGET}'." \
"Was this cloned properly?"
fi
# which size for the qcow2 container?
if [ -z "$DEFAULT_QCOW_SIZE" ]; then
local QCOW_SIZE="10G"
else
local QCOW_SIZE="$DEFAULT_QCOW_SIZE"
fi
# so far so good
pinfo "Creating empty qcow2-container ..."
_STATE='QCOW_CREATING'
qemu-img create -f qcow2 "${CONTAINER_PATH}" "${QCOW_SIZE}" \
|| perror "qemu-img create failed with: $?"
pinfo "Done."
# find usable nbd device
pinfo "Looking for usable nbd device..."
local NBD_DEV="$(find_free_nbd)"
[ -z "${NBD_DEV}" ] && perror "Could not find usable NBD device."
[ -b "${NBD_DEV}" ] || perror "'${NBD_DEV}' is not a block device!"
pinfo "Exporting '${CONTAINER_PATH}' using '${NBD_DEV}'..."
_STATE='QCOW_NBD_CONNECTING'
qemu-nbd -c "${NBD_DEV}" "${CONTAINER_PATH}" || \
perror "qemu-nbd failed with: $?"
pinfo "Done."
# which filesystem for the qcow2 container?
if [ -z "$DEFAULT_QCOW_FS" ]; then
local QCOW_FS="ext4"
else
# check if we have mkfs helper
which "mkfs.$DEFAULT_QCOW_FS" &>/dev/null || \
perror "Could not find 'mkfs.$DEFAULT_QCOW_FS'."
local QCOW_FS="$DEFAULT_QCOW_FS"
fi
pinfo "Creating '${QCOW_FS}' filesystem on '${CONTAINER_PATH}'..."
_STATE='QCOW_FSING'
mkfs."${QCOW_FS}" "${NBD_DEV}" || perror "mkfs failed with: $?"
pinfo "Done."
# prepare NBD mount directory and check state to be safe
local NBD_MNT="$(mktemp -d)"
[ ! -d "${NBD_MNT}" ] && \
perror "Making temporary dir for mounting '$NBD_DEV' failed."
is_dir_empty ${NBD_MNT} || \
perror "'${NBD_MNT}' not empty. Refusing to mount ${NBD_DEV} to it."
pinfo "Mounting '${NBD_DEV}' to '${NBD_MNT}'..."
_STATE='QCOW_MOUNTING'
mount "${NBD_DEV}" "${NBD_MNT}" || perror "Mount failed with: $?"
pinfo "Done."
# copy files from the stage4 directory to the mounted qcow2-container
pinfo "Copying '${RSYNC_TARGET}' to '${NBD_MNT}'..."
_STATE='QCOW_COPYING'
cp -ra "${RSYNC_TARGET}"/* "${NBD_MNT}" || perror "Copying failed with: $?"
pinfo "Done."
pinfo "Cleaning up..."
_STATE='QCOW_CLEANING'
umount "${NBD_MNT}" || pwarning "Could not unmount '${NBD_MNT}'."
rmdir "${NBD_MNT}" || pwarning "Could not remove '${NBD_MNT}'."
qemu-nbd -d "${NBD_DEV}" || pwarning "Could not disconnect '${NBD_DEV}'."
_STATE='QCOW_DONE'
pinfo "Exporting '${RSYNC_TARGET}' to '${CONTAINER_PATH}' completed."
}
# helper to find an unused nbd device
# Usage:
# find_free_nbd
# Echoes the name of the free device to stdout, empty string otherwise.
find_free_nbd() {
local nbd_size=0
for nbd_id in {0..15}; do
[ -b "/dev/nbd${nbd_id}" ] || continue
[ -r "/sys/block/nbd${nbd_id}/size" ] || continue
nbd_size=$(cat /sys/block/nbd${nbd_id}/size)
[ $nbd_size -eq 0 ] && echo "/dev/nbd${nbd_id}" && break
done
echo ""
}
