# ------------------------------------------------------------------------------ # # Functions for OpenSLX-NG # # ------------------------------------------------------------------------------ # # check if we have our environment variables to check # if we actually got sourced by the main script if [ -z "${SELF_PID}" -o -z "${ROOT_DIR}" ]; then # not using perror, since we probably don't have it. echo "Neither SELF_PID nor ROOT_DIR is set. Was this included by OpenSLX-NG?" exit 1 fi # ------------------------------------------------------------------------------ # # General helper functions # # ------------------------------------------------------------------------------ banner () { echo -e "\033[38;5;196m\t"' .__ ' echo -e "\033[38;5;196m\t"' ____ ______ ____ ____ _____| | ___ ___ ' echo -e "\033[38;5;202m\t"' / _ \\\\____ \\_/ __ \\ / \\ / ___/ | \\ \\/ / ' echo -e "\033[38;5;208m\t"'( <_> ) |_> > ___/| | \\\\___ \\| |__> < ' echo -e "\033[38;5;214m\t"' \\____/| __/ \\___ >___| /____ >____/__/\\_ \\ ' echo -e "\033[38;5;220m\t"' |__| \\/ \\/ \\/ \\/ ' echo -e "\033[38;5;220m\t" echo -e "\033[38;5;226m\t ** OpenSLX Project // 2015 **" echo -e "\033[38;5;232m\t http://lab.openslx.org/" echo -e "\033[00m" } pinfo() { echo -e "\033[38;5;10m[info]\033[0m $@" } pwarning() { echo -e "\033[38;5;11m[warning]\033[0m $@" } perror() { echo -e "\033[38;5;9m[error]\033[0m $@" kill "$SELF_PID" exit 1 } print_usage() { pinfo "USAGE:" pinfo "$ARG0 " pinfo "\t\tActions: '--clone', '--package'" pinfo "" pinfo "CLONING:" pinfo "$ARG0 --clone --host [--syncdir ]" pinfo "\t\tIf not specified, --syncdir = './builds//stage4'" pinfo "" pinfo "PACKAGING:" pinfo "$ARG0 --package --syncdir --container " pinfo "$ARG0 --package --host --container " pinfo "\t\tIf is specified, --syncdir = './builds//stage4'" pinfo "$ARG0 --package --host --syncdir --container " pinfo "\t\tIf both are specified, --syncdir is used." kill "$SELF_PID" exit 1 } # # helper to parse the command line arguments and fill the environment # with the parameters given. Note that checks for validity happens # in the respective functions, we only parse here. read_params() { # initialize global variables declare -g FORCE=0 unset ACTION REMOTE_HOST CONTAINER_PATH RSYNC_TARGET # handle rest of arguments while [ "$#" -gt "0" ]; do local PARAM="$1" shift # options to current target if [[ "$PARAM" == --* ]]; then case "$PARAM" in --clone) declare -rg ACTION="CLONE" ;; --package) declare -rg ACTION="PACKAGE" ;; --host) if [ -z "$1" ]; then pwarning "'--host' requires a host as parameter." print_usage && exit 1 else if [[ "$1" == --* ]]; then local _msg="A host should not start with '--'." _msg="$_msg Parameter for '--host' missing?" perror $_msg fi declare -rg REMOTE_HOST="$1" shift fi continue ;; --container) if [ -z "$1" ]; then pwarning "'--container' requires a path as parameter." print_usage && exit 1 else if [[ "$1" == --* ]]; then local _msg="A path should not start with '--'." _msg="$_msg Parameter for '--container' missing?" perror $_msg fi declare -rg CONTAINER_PATH="$1" shift fi continue ;; --syncdir) if [ -z "$1" ]; then pwarning "'--syncdir' requires a path as parameter." print_usage && exit 1 else if [[ "$1" == --* ]]; then local _msg="A path should not start with '--'." _msg="$_msg Parameter for '--syncdir' missing?" perror $_msg fi declare -rg RSYNC_TARGET="$1" shift fi continue ;; --force) declare -rg FORCE=1 ;; *) pwarning "Unknown flag: $PARAM" print_usage && exit 1 ;; esac continue fi done } # helper function trapped on SIGTERM/SIGINT # Usage: do not use as is cleanexit() { trap '' SIGINT SIGTERM # from now on, ignore INT and TERM pwarning "SIGINT/SIGTERM triggered - cleaning up ..." [ -z "${_STATE}" ] && perror "'_STATE' not set, this is bad." case "${_STATE}" in SYNC_DONE|QCOW_DONE) # we are happy pwarning "SIGINT/SIGTERM received, but everything seems fine. Check it!" exit 0 ;; BLACKLISTING) # creation of blacklists failed # TODO do what? ;; SYNCING) # error during rsync, create the .stage4 file again [ -z "${RSYNC_TARGET}" ] && \ perror "RSYNC_TARGET not set, this should not happen." if [ ! -e "${RSYNC_TARGET}/.stage4" ]; then pwarning "'.stage4' flag was lost during rsync, restoring it." touch "${RSYNC_TARGET}/.stage4" fi ;; QCOW_CREATING) # qemu-img failed. Just remove the container if its there if [ -n "${CONTAINER_PATH}" -a -e "${CONTAINER_PATH}" ]; then rm -f "${CONTAINER_PATH}" || \ pwarning "Could not remove '${CONTAINER_PATH}'." fi ;; QCOW_NBD_CONNECTING) # qemu-nbd failed if [ -n "${NBD_DEV}" ]; then qemu-nbd -d "${NBD_DEV}" && \ pwarning "Could not disconnect '${NBD_DEV}'." fi ;; QCOW_FSING) # mkfs failed, disconnect and remove container if [ -n "${NBD_DEV}" ]; then qemu-nbd -d "${NBD_DEV}" && \ pwarning "Could not disconnect '${NBD_DEV}'." fi if [ -n "${CONTAINER_PATH}" -a -e "${CONTAINER_PATH}" ]; then rm -f "${CONTAINER_PATH}" || \ pwarning "Could not remove '${CONTAINER_PATH}'." fi ;; QCOW_MOUNTING) # mounting failed: # umount, disconnect and remove container and mount point if [ -n "${NBD_MNT}" ]; then umount "${NBD_MNT}" || pwarning "Could not umount '${NBD_MNT}'." rmdir "${NBD_MNT}" || pwarning "Could not rmdir '${NBD_MNT}'." fi if [ -n "${NBD_DEV}" ]; then qemu-nbd -d "${NBD_DEV}" && \ perror "Could not disconnect '${NBD_DEV}'." fi if [ -n "${CONTAINER_PATH}" -a -e "${CONTAINER_PATH}" ]; then rm -f "${CONTAINER_PATH}" || \ perror "Could not remove '${CONTAINER_PATH}'." fi ;; QCOW_COPYING) # rare case, should not happen ;; QCOW_CLEANING) # should also not happen ;; *) pwarning "Unknown state: ${_STATE}" esac # still here? then we ran into some error exit 1 } # helper to validate an ip # Usage: # valid_ip # Returns 0 if valid, 1 otherwise. valid_ip() { local ip=$1 local stat=1 if [[ $ip =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then OIFS=$IFS IFS='.' ip=($ip) IFS=$OIFS [[ ${ip[0]} -le 255 && ${ip[1]} -le 255 \ && ${ip[2]} -le 255 && ${ip[3]} -le 255 ]] stat=$? fi return $stat } # helper to check whether a given host is valid # Usage: # check_host # Returns 0 if valid, 1 otherwise check_host() { local HOST="$1" [ -z "$HOST" ] && return 1 # check if its a valid IP or a valid hostname valid_ip "$HOST" && return 0 host -W 2 "$HOST" && return 0 # still here? then fail return 1 } # helper to check if a dir is empty or not # Usage: # dir_empty # Returns 0 if empty, 1 otherwise is_dir_empty() { [ $# -ne 1 ] && perror "$0 requires directory as paramter, none given." local _dir="$1" [ -d "$_dir" ] || return 1 [ -n "$(ls -A $_dir)" ] && return 1 || return 0 } # helper to ask user for confirmation # Usage: # user_confirm # Return 0 if confirmed, 1 otherwise user_confirm() { [ $# -ne 1 ] && perror "$0 requires the question as first argument." pinfo "$1 [Y/n]" local _input read _input [ "x${_input}" == "x" -o "x${_input}" == "xy" ] && return 0 || return 1 } # ------------------------------------------------------------------------------ # # Stage4 related functions # # ------------------------------------------------------------------------------ # # Helper to generate a stage4 export for a remote machine per rsync. # Usage: # clone_stage4 # # Note: this functions requires REMOTE_HOST and RSYNC_TARGET to be set. clone_stage4() { # REMOTE_HOST valid? [ -z "$REMOTE_HOST" ] && pwarning "REMOTE_HOST not set. Use '--host'." && print_usage if ! check_host "$REMOTE_HOST"; then # invalid, abort pwarning "'$REMOTE_HOST' is neither an IP nor a known hostname." print_usage fi # REMOTE_HOST is valid, use it as the base for our build files # set BUILD_DIR relative to the ROOT_DIR for the REMOTE_HOST declare -rg BUILD_DIR="${ROOT_DIR}/builds/$REMOTE_HOST" mkdir -p "${BUILD_DIR}" # RSYNC_TARGET set? if [ -z "$RSYNC_TARGET" ]; then pwarning "RSYNC_TARGET not set. Assuming local mode." pinfo "Using '${BUILD_DIR}/stage4'" declare -rg RSYNC_TARGET="${BUILD_DIR}/stage4" fi # check if RSYNC_TARGET is valid if [ -d "${RSYNC_TARGET}" ]; then # does it have the '.stage4' flag? [ ! -e "${RSYNC_TARGET}/.stage4" ] && \ perror "'${RSYNC_TARGET}' exists, but no '.stage4' flag found." \ "Refusing to rsync there." else # not a directory, create it and set the .stage4 flag mkdir -p "${RSYNC_TARGET}" touch "${RSYNC_TARGET}/.stage4" fi # mark state _STATE='BLACKLISTING' local EXCLUDE="$BUILD_DIR/exclude-stage4" local INCLUDE="$BUILD_DIR/include-stage4" pinfo "Building rsync include/exclude files for building stage4...." echo "## Exclude file for stage4 of $REMOTE_HOST" > "$EXCLUDE" echo "## Include file for stage4 of $REMOTE_HOST" > "$INCLUDE" for FILE in $(find "${ROOT_DIR}"/blacklists/*/ -type f); do echo "## From $FILE" >> "$EXCLUDE" echo "## From $FILE" >> "$INCLUDE" grep '^-' "$FILE" >> "$EXCLUDE" grep '^+' "$FILE" >> "$INCLUDE" done pinfo "Done." # prepare rsync's options if [ -z "$DEFAULT_RSYNC_OPTS" ]; then local RSYNC_OPTS="-e ssh -c arcfour -oStrictHostKeyChecking=no" else local RSYNC_OPTS="$DEFAULT_RSYNC_OPTS" fi local RSYNC_SOURCE="root@$REMOTE_HOST:/" _STATE='SYNCING' # run rsync with the exclude/include lists created earlier cat "$INCLUDE" "$EXCLUDE" | \ rsync --verbose \ --archive \ --delete \ --delete-excluded \ --numeric-ids \ --exclude-from=- \ "${RSYNC_OPTS}" \ "${RSYNC_SOURCE}" \ "${RSYNC_TARGET}" \ || perror "rsync from '${RSYNC_SOURCE}' to '${RSYNC_TARGET}' failed." ## TODO real exit code handling pinfo "Cloning '${REMOTE_HOST}' to '${RSYNC_TARGET}' succeeded." _STATE='SYNC_DONE' touch "${RSYNC_TARGET}/.stage4" return 0 } # helper to build a qcow2 container from a stage4 sync directory # Usage: # pack_qcow2 # # Note that this requires CONTAINER_PATH to be set. # RSYNC_TARGET is either the path given through the option '--syncdir' # or the standard local path '$ROOT_DIR/builds/$REMOTE_HOST/stage4' # is assumed to be our rsync destination directory. pack_qcow2() { # CONTAINER_PATH valid? [ -z "$CONTAINER_PATH" ] && \ pwarning "CONTAINER_PATH not set. Use '--container'." && print_usage [ -d "$CONTAINER_PATH" ] && perror "Path to container can not be a directory!" if [ -f "$CONTAINER_PATH" ]; then if [ $FORCE -eq 0 ]; then perror "Container file already exists. Use '--force' to overwrite it." else # force removal rm -f "$CONTAINER_PATH" || perror "Could not remove '$CONTAINER_PATH'" pinfo "Removed old '$CONTAINER_PATH'." fi fi # RSYNC_TARGET valid? if [ -z "$RSYNC_TARGET" ]; then # if not RSYNC_TARGET was specified, we must have REMOTE_HOST # or we do not know which stage4 we are supposed to package [ -z "$REMOTE_HOST" ] && \ pwarning "Need either '--syncdir' or '--host'!" && \ print_usage check_host "$REMOTE_HOST" || perror "Given host invalid." pwarning "RSYNC_TARGET not set. Assuming local mode." local RSYNC_TARGET_CANDIDATE="${ROOT_DIR}/builds/${REMOTE_HOST}/stage4" if [ ! -d "$RSYNC_TARGET_CANDIDATE" ]; then pwarning "Local stage4 sync not found at '${RSYNC_TARGET_CANDIDATE}'" pwarning "Did you sync with '--syncdir' set? Then use that :)" perror "Stage4 to package not found." fi is_dir_empty "$RSYNC_TARGET_CANDIDATE" && \ perror "Appears to be empty, did you clone?" pinfo "Found '$RSYNC_TARGET_CANDIDATE', using it." declare -rg RSYNC_TARGET="$RSYNC_TARGET_CANDIDATE" fi # more sanity checks [ ! -d "$RSYNC_TARGET" ] && perror "'$RSYNC_TARGET' not a directory!" is_dir_empty "$RSYNC_TARGET" && \ perror "'$RSYNC_TARGET' appears to be empty. Did you clone?" # the ultimative check if [ ! -e "${RSYNC_TARGET}/.stage4" ]; then perror "No '.stage4' flag found in '${RSYNC_TARGET}'." \ "Was this cloned properly?" fi # which size for the qcow2 container? if [ -z "$DEFAULT_QCOW_SIZE" ]; then local QCOW_SIZE="10G" else local QCOW_SIZE="$DEFAULT_QCOW_SIZE" fi # so far so good pinfo "Creating empty qcow2-container ..." _STATE='QCOW_CREATING' qemu-img create -f qcow2 "${CONTAINER_PATH}" "${QCOW_SIZE}" \ || perror "qemu-img create failed with: $?" pinfo "Done." # find usable nbd device pinfo "Looking for usable nbd device..." local NBD_DEV="$(find_free_nbd)" [ -z "${NBD_DEV}" ] && perror "Could not find usable NBD device." [ -b "${NBD_DEV}" ] || perror "'${NBD_DEV}' is not a block device!" pinfo "Exporting '${CONTAINER_PATH}' using '${NBD_DEV}'..." _STATE='QCOW_NBD_CONNECTING' qemu-nbd -c "${NBD_DEV}" "${CONTAINER_PATH}" || \ perror "qemu-nbd failed with: $?" pinfo "Done." # which filesystem for the qcow2 container? if [ -z "$DEFAULT_QCOW_FS" ]; then local QCOW_FS="ext4" else # check if we have mkfs helper which "mkfs.$DEFAULT_QCOW_FS" &>/dev/null || \ perror "Could not find 'mkfs.$DEFAULT_QCOW_FS'." local QCOW_FS="$DEFAULT_QCOW_FS" fi pinfo "Creating '${QCOW_FS}' filesystem on '${CONTAINER_PATH}'..." _STATE='QCOW_FSING' mkfs."${QCOW_FS}" "${NBD_DEV}" || perror "mkfs failed with: $?" pinfo "Done." # prepare NBD mount directory and check state to be safe local NBD_MNT="$(mktemp -d)" [ ! -d "${NBD_MNT}" ] && \ perror "Making temporary dir for mounting '$NBD_DEV' failed." is_dir_empty ${NBD_MNT} || \ perror "'${NBD_MNT}' not empty. Refusing to mount ${NBD_DEV} to it." pinfo "Mounting '${NBD_DEV}' to '${NBD_MNT}'..." _STATE='QCOW_MOUNTING' mount "${NBD_DEV}" "${NBD_MNT}" || perror "Mount failed with: $?" pinfo "Done." # copy files from the stage4 directory to the mounted qcow2-container pinfo "Copying '${RSYNC_TARGET}' to '${NBD_MNT}'..." _STATE='QCOW_COPYING' cp -ra "${RSYNC_TARGET}"/* "${NBD_MNT}" || perror "Copying failed with: $?" pinfo "Done." pinfo "Cleaning up..." _STATE='QCOW_CLEANING' umount "${NBD_MNT}" || pwarning "Could not unmount '${NBD_MNT}'." rmdir "${NBD_MNT}" || pwarning "Could not remove '${NBD_MNT}'." qemu-nbd -d "${NBD_DEV}" || pwarning "Could not disconnect '${NBD_DEV}'." _STATE='QCOW_DONE' pinfo "Exporting '${RSYNC_TARGET}' to '${CONTAINER_PATH}' completed." } # helper to find an unused nbd device # Usage: # find_free_nbd # Echoes the name of the free device to stdout, empty string otherwise. find_free_nbd() { local nbd_size=0 for nbd_id in {0..15}; do [ -b "/dev/nbd${nbd_id}" ] || continue [ -r "/sys/block/nbd${nbd_id}/size" ] || continue nbd_size=$(cat /sys/block/nbd${nbd_id}/size) [ $nbd_size -eq 0 ] && echo "/dev/nbd${nbd_id}" && break done echo "" }