From 0ca5799f2e65cc508b45d3f3a290434e529776c3 Mon Sep 17 00:00:00 2001 From: Jonathan Bauer Date: Fri, 29 May 2015 18:23:31 +0200 Subject: finally in a working state. See usage. --- packager/openslx | 48 ++---- packager/openslx.config | 8 + packager/openslx.functions | 383 +++++++++++++++++++++++++++++++++++++-------- 3 files changed, 341 insertions(+), 98 deletions(-) create mode 100644 packager/openslx.config mode change 100755 => 100644 packager/openslx.functions (limited to 'packager') diff --git a/packager/openslx b/packager/openslx index 6a75fc47..a71a1db6 100755 --- a/packager/openslx +++ b/packager/openslx @@ -8,25 +8,6 @@ # The next generation toolkit to build a stateless linux system bootable # over the network. Composed of dracut-built initramfs and a full system clone. # -# ------------------------------------------------------------------------------ -# -# OpenSLX-NG Functions -# -# ------------------------------------------------------------------------------ -pinfo() { - echo -e "\033[38;5;10m[info]\033[0m $@" -} - -perror() { - echo -e "\033[38;5;9m[error]\033[0m $@" - kill "$SELF_PID" - exit 1 -} - -print_usage() { - perror "$ARG0 " -} - # ------------------------------------------------------------------------------ # # OpenSLX-NG Main Code @@ -38,11 +19,15 @@ declare -rg SELF="$(readlink -f "$ARG0")" declare -rg ROOT_DIR="$(dirname "${SELF}")" declare -rg SELF_PID="$$" +# let's keep it simple for now, just source our config/functions file +. "${ROOT_DIR}/$ARG0".functions || perror "Could not source functions." +. "${ROOT_DIR}/$ARG0".config || perror "Could not source config." + # root check if [ "$(id -u)" -ne 0 ]; then perror "ERROR: You need to be root to use this toolkit." else - pinfo "" + banner fi # do we even have tools? @@ -50,19 +35,14 @@ for TOOL in qemu-img qemu-nbd mkfs.ext4; do which $TOOL &>/dev/null || perror "Could not find '$TOOL'." done -# let's keep it simple for now, just source our main functions file -. "${ROOT_DIR}/$ARG0".functions || perror "Could not source functions." - -# ok, now check that we have 3 args -[ $# -ne 3 ] && print_usage -# even though functions check their parameter, we also do it to be safe. -valid_ip "$1" || perror "'$1' is not a valid IP adress, aborting..." -[ ! -d "$2" ] || perror "'$2' already exists, aborting..." -[ ! -f "$3" ] || perror "'$3' already exists, aborting..." +read_params $@ +if [ "x$ACTION" == "xCLONE" ]; then + clone_stage4 || perror "Cloning stage4 failed with: $?" +elif [ "x$ACTION" == "xPACKAGE" ]; then + pack_qcow2 || perror "Packing as QCoW2 failed with: $?" +else + pwarning "No action given." + print_usage +fi -# all good, let's start -pinfo " ## RUNNING ##" -clone_stage4 $1 $2 || perror "Failed to clone stage4 with: $1 $2" -export_qcow2 $2 $3 || perror "Failed to export stage4 with: $2 $3" -pinfo " ## COMPLETE ##" diff --git a/packager/openslx.config b/packager/openslx.config new file mode 100644 index 00000000..00fc5501 --- /dev/null +++ b/packager/openslx.config @@ -0,0 +1,8 @@ +# default options used by rsync +DEFAULT_RSYNC_OPTS="-e ssh -c arcfour -oStrictHostKeyChecking=no" + +# default size for the qcow2-container (the one containing diffs) +DEFAULT_QCOW_SIZE="10G" + +# default filesystem type for the qcow2-container +DEFAULT_QCOW_FS="ext4" diff --git a/packager/openslx.functions b/packager/openslx.functions old mode 100755 new mode 100644 index 5fabdbe0..867e252d --- a/packager/openslx.functions +++ b/packager/openslx.functions @@ -12,22 +12,232 @@ if [ -z "${SELF_PID}" -o -z "${ROOT_DIR}" ]; then exit 1 fi +# ------------------------------------------------------------------------------ +# +# General helper functions +# +# ------------------------------------------------------------------------------ +banner () { + echo -e "\033[38;5;196m\t"' .__ ' + echo -e "\033[38;5;196m\t"' ____ ______ ____ ____ _____| | ___ ___ ' + echo -e "\033[38;5;202m\t"' / _ \\\\____ \\_/ __ \\ / \\ / ___/ | \\ \\/ / ' + echo -e "\033[38;5;208m\t"'( <_> ) |_> > ___/| | \\\\___ \\| |__> < ' + echo -e "\033[38;5;214m\t"' \\____/| __/ \\___ >___| /____ >____/__/\\_ \\ ' + echo -e "\033[38;5;220m\t"' |__| \\/ \\/ \\/ \\/ ' + echo -e "\033[38;5;220m\t" + echo -e "\033[38;5;226m\t ** OpenSLX Project // 2015 **" + echo -e "\033[38;5;232m\t http://lab.openslx.org/" + echo -e "\033[00m" +} + +pinfo() { + echo -e "\033[38;5;10m[info]\033[0m $@" +} + +pwarning() { + echo -e "\033[38;5;11m[warning]\033[0m $@" +} + +perror() { + echo -e "\033[38;5;9m[error]\033[0m $@" + kill "$SELF_PID" + exit 1 +} + +print_usage() { + pinfo "USAGE:" + pinfo "$ARG0 " + pinfo "\t\tActions: '--clone', '--package'" + pinfo "" + pinfo "CLONING:" + pinfo "$ARG0 --clone --host [--syncdir ]" + pinfo "\t\tIf not specified, --syncdir = './builds//stage4'" + pinfo "" + pinfo "PACKAGING:" + pinfo "$ARG0 --package --syncdir --container " + pinfo "$ARG0 --package --host --container " + pinfo "\t\tIf is specified, --syncdir = './builds//stage4'" + pinfo "$ARG0 --package --host --syncdir --container " + pinfo "\t\tIf both are specified, --syncdir is used." + kill "$SELF_PID" + exit 1 +} +# +# helper to parse the command line arguments and fill the environment +# with the parameters given. Note that checks for validity happens +# in the respective functions, we only parse here. +read_params() { + # initialize global variables + declare -g FORCE=0 + unset ACTION REMOTE_HOST CONTAINER_PATH RSYNC_TARGET + + # handle rest of arguments + while [ "$#" -gt "0" ]; do + local PARAM="$1" + shift + + # options to current target + if [[ "$PARAM" == --* ]]; then + case "$PARAM" in + --clone) + declare -rg ACTION="CLONE" + ;; + --package) + declare -rg ACTION="PACKAGE" + ;; + --host) + if [ -z "$1" ]; then + pwarning "'--host' requires a host as parameter." + print_usage && exit 1 + else + if [[ "$1" == --* ]]; then + local _msg="A host should not start with '--'." + _msg="$_msg Parameter for '--host' missing?" + perror $_msg + fi + declare -rg REMOTE_HOST="$1" + shift + fi + continue + ;; + --container) + if [ -z "$1" ]; then + pwarning "'--container' requires a path as parameter." + print_usage && exit 1 + else + if [[ "$1" == --* ]]; then + local _msg="A path should not start with '--'." + _msg="$_msg Parameter for '--container' missing?" + perror $_msg + fi + declare -rg CONTAINER_PATH="$1" + shift + fi + continue + ;; + --syncdir) + if [ -z "$1" ]; then + pwarning "'--syncdir' requires a path as parameter." + print_usage && exit 1 + else + if [[ "$1" == --* ]]; then + local _msg="A path should not start with '--'." + _msg="$_msg Parameter for '--syncdir' missing?" + perror $_msg + fi + declare -rg RSYNC_TARGET="$1" + shift + fi + continue + ;; + --force) + declare -rg FORCE=1 + ;; + *) + pwarning "Unknown flag: $PARAM" + print_usage && exit 1 + ;; + esac + continue + fi + done +} + +# helper to validate an ip +# Usage: +# valid_ip +# Returns 0 if valid, 1 otherwise. +valid_ip() { + local ip=$1 + local stat=1 + + if [[ $ip =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then + OIFS=$IFS + IFS='.' + ip=($ip) + IFS=$OIFS + [[ ${ip[0]} -le 255 && ${ip[1]} -le 255 \ + && ${ip[2]} -le 255 && ${ip[3]} -le 255 ]] + stat=$? + fi + return $stat +} + +# helper to check if a dir is empty or not +# Usage: +# dir_empty +# Returns 0 if empty, 1 otherwise +is_dir_empty() { + [ $# -ne 1 ] && perror "$0 requires directory as paramter, none given." + local _dir="$1" + [ -d "$_dir" ] || return 1 + [ -n "$(ls -A $_dir)" ] && return 1 || return 0 +} + +# helper to ask user for confirmation +# Usage: +# user_confirm +# Return 0 if confirmed, 1 otherwise +user_confirm() { + [ $# -ne 1 ] && perror "$0 requires the question as first argument." + pinfo "$1 [Y/n]" + local _input + read _input + [ "x${_input}" == "x" -o "x${_input}" == "xy" ] && return 0 || return 1 +} + +# helper to check whether a given host is valid +# Usage: +# check_host +# Returns 0 if valid, 1 otherwise +check_host() { + local HOST="$1" + [ -z "$HOST" ] && return 1 + # check if its a valid IP or a valid hostname + valid_ip "$HOST" && return 0 + host -W 2 "$HOST" && return 0 + # still here? then fail + return 1 +} +# ------------------------------------------------------------------------------ +# +# Stage4 related functions +# +# ------------------------------------------------------------------------------ +# # Helper to generate a stage4 export for a remote machine per rsync. +# Usage: +# clone_stage4 +# +# Note: this functions requires REMOTE_HOST and RSYNC_TARGET to be set. clone_stage4() { - [ $# -ne 2 ] && perror "$0 " - local REMOTE_HOST="$1" - local TARGET_DIR="$2" - local BUILD_DIR="${ROOT_DIR}/builds/$REMOTE_HOST" - mkdir -p "$TARGET_DIR" - mkdir -p "$BUILD_DIR" + # REMOTE_HOST valid? + [ -z "$REMOTE_HOST" ] && pwarning "REMOTE_HOST not set. Use '--host'." && print_usage + if ! check_host "$REMOTE_HOST"; then + # invalid, abort + pwarning "'$REMOTE_HOST' is neither an IP nor a known hostname." + print_usage + fi + # REMOTE_HOST is valid, use it as the base for our build files + # set BUILD_DIR relative to the ROOT_DIR for the REMOTE_HOST + declare -rg BUILD_DIR="${ROOT_DIR}/builds/$REMOTE_HOST" + mkdir -p "${BUILD_DIR}" + + # RSYNC_TARGET set? + if [ -z "$RSYNC_TARGET" ]; then + pwarning "RSYNC_TARGET not set. Assuming local mode" + pinfo "Using '${BUILD_DIR}/stage4'" + declare -rg RSYNC_TARGET="${BUILD_DIR}/stage4" + fi + mkdir -p "${RSYNC_TARGET}" local EXCLUDE="$BUILD_DIR/exclude-stage4" local INCLUDE="$BUILD_DIR/include-stage4" - pinfo "Building rsync exclude-file for building stage 4...." + pinfo "Building rsync include/exclude files for building stage4...." echo "## Exclude file for stage4 of $REMOTE_HOST" > "$EXCLUDE" echo "## Include file for stage4 of $REMOTE_HOST" > "$INCLUDE" - for FILE in $(find "$ROOT_DIR"/blacklists/*/ -type f); do + for FILE in $(find "${ROOT_DIR}"/blacklists/*/ -type f); do echo "## From $FILE" >> "$EXCLUDE" echo "## From $FILE" >> "$INCLUDE" grep '^-' "$FILE" >> "$EXCLUDE" @@ -36,11 +246,14 @@ clone_stage4() { pinfo "Done." # prepare rsync's options + if [ -z "$DEFAULT_RSYNC_OPTS" ]; then + local RSYNC_OPTS="-e ssh -c arcfour -oStrictHostKeyChecking=no" + else + local RSYNC_OPTS="$DEFAULT_RSYNC_OPTS" + fi local RSYNC_SOURCE="root@$REMOTE_HOST:/" - local RSYNC_OPTS="-e ssh -c arcfour -oStrictHostKeyChecking=no" # run rsync with the exclude/include lists created earlier - pinfo "Cloning via rsync..." cat "$INCLUDE" "$EXCLUDE" | \ rsync --verbose \ --archive \ @@ -50,89 +263,131 @@ clone_stage4() { --exclude-from=- \ "${RSYNC_OPTS}" \ "${RSYNC_SOURCE}" \ - "${TARGET_DIR}" \ - || perror "rsync from '${RSYNC_SOURCE}' to '${TARGET_DIR}' failed." - pinfo "Cloning '${REMOTE_HOST}' to '${TARGET_DIR}' succeeded." + "${RSYNC_TARGET}" \ + || perror "rsync from '${RSYNC_SOURCE}' to '${RSYNC_TARGET}' failed." + ## TODO real exit code handling + pinfo "Cloning '${REMOTE_HOST}' to '${RSYNC_TARGET}' succeeded." return 0 } -# helper to build a qcow2 container from a stage4 sync directory. -# required tools: qemu-img, qemu-nbd, nbd, mkfs.ext4 -export_qcow2() { - # enough args? - [ $# -ne 2 ] && perror "Usage: $0 " - # $1 valid? - [ ! -d $1 ] && perror "First argument not a directory!" - # $2 should not be a dir or strange things will happen... - [ -d $2 ] && perror "Target file can not be a directory!" - - # $1 is dir, strip trailing slash if there is one - local STAGE4_DIR="${1%/}" - local TARGET_FILE="$2" - if [ -e $TARGET_FILE ]; then - pinfo "Removing old target: $TARGET_FILE" - rm $target || perror "Could not remove '$TARGET_FILE'" +# helper to build a qcow2 container from a stage4 sync directory +# Usage: +# pack_qcow2 +# +# Note that this requires CONTAINER_PATH to be set. +# RSYNC_TARGET is either the path given through the option '--syncdir' +# or the standard local path '$ROOT_DIR/builds/$REMOTE_HOST/stage4' +# is assumed to be our rsync destination directory. +pack_qcow2() { + # CONTAINER_PATH valid? + [ -z "$CONTAINER_PATH" ] && \ + pwarning "CONTAINER_PATH not set. Use '--container'." && print_usage + [ -d "$CONTAINER_PATH" ] && perror "Path to container can not be a directory!" + if [ -f "$CONTAINER_PATH" ]; then + if [ $FORCE -eq 0 ]; then + perror "Container file already exists. Use '--force' to overwrite it." + else + # force removal + rm -f "$CONTAINER_PATH" || perror "Could not remove '$CONTAINER_PATH'" + pinfo "Removed old '$CONTAINER_PATH'." + fi + fi + + # RSYNC_TARGET valid? + if [ -z "$RSYNC_TARGET" ]; then + # if not RSYNC_TARGET was specified, we must have REMOTE_HOST + # or we do not know which stage4 we are supposed to package + [ -z "$REMOTE_HOST" ] && \ + pwarning "Need either '--syncdir' or '--host'!" && \ + print_usage + check_host "$REMOTE_HOST" || perror "Given host invalid." + pwarning "RSYNC_TARGET not set. Assuming local mode." + local RSYNC_TARGET_CANDIDATE="${ROOT_DIR}/builds/${REMOTE_HOST}/stage4" + if [ ! -d "$RSYNC_TARGET_CANDIDATE" ]; then + pwarning "Local stage4 sync not found at '${RSYNC_TARGET_CANDIDATE}'" + pwarning "Did you sync with '--syncdir' set? Then use that :)" + perror "Stage4 to package not found." + fi + is_dir_empty "$RSYNC_TARGET_CANDIDATE" && \ + perror "Appears to be empty, did you clone?" + pinfo "Found '$RSYNC_TARGET_CANDIDATE', using it." + declare -rg RSYNC_TARGET="$RSYNC_TARGET_CANDIDATE" fi + # more sanity checks + [ ! -d "$RSYNC_TARGET" ] && perror "'$RSYNC_TARGET' not a directory!" + is_dir_empty "$RSYNC_TARGET" && \ + perror "'$RSYNC_TARGET' appears to be empty. Did you clone?" + + # which size for the qcow2 container? + if [ -z "$DEFAULT_QCOW_SIZE" ]; then + local QCOW_SIZE="10G" + else + local QCOW_SIZE="$DEFAULT_QCOW_SIZE" + fi # so far so good pinfo "Creating empty qcow2-container ..." - qemu-img create -f qcow2 $TARGET_FILE 10G \ + qemu-img create -f qcow2 "${CONTAINER_PATH}" "${QCOW_SIZE}" \ || perror "qemu-img create failed with: $?" pinfo "Done." # find usable nbd device pinfo "Looking for usable nbd device..." - local NBD_ID="$(find_free_nbd)" - local NBD_DEV="/dev/nbd${NBD_ID}" - [ -b "$NBD_DEV" ] || perror "'$NBD_DEV' is not a block device!" - pinfo "Exporting '${TARGET_FILE}' using '${NBD_DEV}'..." - qemu-nbd -c ${NBD_DEV} ${TARGET_FILE} || perror "qemu-nbd failed with: $?" + local NBD_DEV="$(find_free_nbd)" + [ -z "${NBD_DEV}" ] && perror "Could not find usable NBD device." + [ -b "${NBD_DEV}" ] || perror "'${NBD_DEV}' is not a block device!" + pinfo "Exporting '${CONTAINER_PATH}' using '${NBD_DEV}'..." + qemu-nbd -c "${NBD_DEV}" "${CONTAINER_PATH}" || \ + perror "qemu-nbd failed with: $?" pinfo "Done." - pinfo "Creating ext4 filesystem on '${TARGET_FILE}'..." - mkfs.ext4 "${NBD_DEV}" || perror "mkfs failed with: $?" + # which filesystem for the qcow2 container? + if [ -z "$DEFAULT_QCOW_FS" ]; then + local QCOW_FS="ext4" + else + # check if we have mkfs helper + which "mkfs.$DEFAULT_QCOW_FS" &>/dev/null || \ + perror "Could not find 'mkfs.$DEFAULT_QCOW_FS'." + local QCOW_FS="$DEFAULT_QCOW_FS" + fi + pinfo "Creating '${QCOW_FS}' filesystem on '${CONTAINER_PATH}'..." + mkfs."${QCOW_FS}" "${NBD_DEV}" || perror "mkfs failed with: $?" pinfo "Done." - # mount NBD_DEV - mkdir ${BUILD_DIR}/mnt - pinfo "Mounting '${NBD_DEV}' to '${BUILD_DIR}/mnt'..." - mount ${NBD_DEV} ${BUILD_DIR}/mnt || perror "Mount failed with: $?" + # prepare NBD mount directory and check state to be safe + local NBD_MNT="$(mktemp -d)" + [ ! -d "${NBD_MNT}" ] && \ + perror "Making temporary dir for mounting '$NBD_DEV' failed." + is_dir_empty ${NBD_MNT} || \ + perror "'${NBD_MNT}' not empty. Refusing to mount ${NBD_DEV} to it." + + pinfo "Mounting '${NBD_DEV}' to '${NBD_MNT}'..." + mount "${NBD_DEV}" "${NBD_MNT}" || perror "Mount failed with: $?" pinfo "Done." # copy files from the stage4 directory to the mounted qcow2-container - pinfo "Copying '${STAGE4_DIR}' to '${BUILD_DIR}/mnt'..." - cp -ra "${STAGE4_DIR}"/* "${BUILD_DIR}"/mnt || perror "Copying failed with: $?" + pinfo "Copying '${RSYNC_TARGET}' to '${NBD_MNT}'..." + cp -ra "${RSYNC_TARGET}"/* "${NBD_MNT}" || perror "Copying failed with: $?" pinfo "Done." pinfo "Cleaning up..." - umount "${BUILD_DIR}/mnt" || perror "Could not unmount '${BUILD_DIR}/mnt'." - qemu-nbd -d /dev/nbd${nbd_id} || perror "Could not disconnect '${NBD_DEV}'." - pinfo "Exporting '${STAGE4_DIR}' to '${TARGET_FILE}' completed." + umount "${NBD_MNT}" || pwarning "Could not unmount '${NBD_MNT}'." + rmdir "${NBD_MNT}" || pwarning "Could not remove '${NBD_MNT}'." + qemu-nbd -d "${NBD_DEV}" || pwarning "Could not disconnect '${NBD_DEV}'." + pinfo "Exporting '${RSYNC_TARGET}' to '${CONTAINER_PATH}' completed." } # helper to find an unused nbd device +# Usage: +# find_free_nbd +# Echoes the name of the free device to stdout, empty string otherwise. find_free_nbd() { local nbd_size=0 for nbd_id in {0..15}; do [ -b "/dev/nbd${nbd_id}" ] || continue - [ ! -e "/sys/block/nbd${nbd_id}/size" ] || continue + [ -r "/sys/block/nbd${nbd_id}/size" ] || continue nbd_size=$(cat /sys/block/nbd${nbd_id}/size) - [ $nbd_size -eq 0 ] && break + [ $nbd_size -eq 0 ] && echo "/dev/nbd${nbd_id}" && break done - return ${nbd_id} + echo "" } -# helper to validate an ip -valid_ip() { - local ip=$1 - local stat=1 - if [[ $ip =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then - OIFS=$IFS - IFS='.' - ip=($ip) - IFS=$OIFS - [[ ${ip[0]} -le 255 && ${ip[1]} -le 255 \ - && ${ip[2]} -le 255 && ${ip[3]} -le 255 ]] - stat=$? - fi - return $stat -} -- cgit v1.2.3-55-g7522