initial support for different qcow handlers

3 files changed, 317 insertions, 96 deletions

diff --git a/builder/build-initramfs.sh b/builder/build-initramfs.sh
index 66dbf172..e10ac67c 100755
--- a/builder/build-initramfs.sh
+++ b/builder/build-initramfs.sh
@@ -22,21 +22,20 @@
 # Note that you only get very necessary output until you provide "--verbose" as
 # commandline options.
 declare -rg _root_dir="$(readlink -f $(dirname ${BASH_SOURCE[0]}))"
+declare -rg _repo_dir="${_root_dir}/systemd-init.git"
 declare -rg _git_source="git://git.openslx.org/openslx-ng/systemd-init.git"
 declare -rg _git_branch="master"
 ## region ensure presence of needed dependencies
 set -o errexit
-_needed_location="${_root_dir}/modules.d"
-if ! [[ -d "$_needed_location" ]]; then
-    echo "The dracut modules aren't available, loading them."
+if [ ! -e "$_repo_dir" ]; then
+    echo "Missing dracut modules repository, loading them."
     if ! hash git; then
         echo "Needed dependency \"git\" isn't available."
-        echo "Please install \"git\" or provide the repositories data structure in \"${_root_dir}\"."
+        echo "Please install \"git\" or provide the main repository in \"${_repo_dir}\"."
     fi
-    _temporary_repository_location="$(mktemp --directory)"
     git clone --branch "$_git_branch" --single-branch --depth 1 \
-       "$_git_source" "$_temporary_repository_location"
-    pushd "$_temporary_repository_location"
+       "$_git_source" "${_repo_dir}"
+    pushd "${_repo_dir}"
     git submodule init
     # try to clone submodules as shallowy as possible, since we cannot just
     # use '--depth 1' on submodules residing on non-master branches...
@@ -46,10 +45,11 @@ if ! [[ -d "$_needed_location" ]]; then
         branch="$(git config -f .gitmodules --get submodule.${mod}.branch)"
         commit="$(git submodule status $path | grep -oE '[0-9a-f]{40}')"
         depth_arg=("--shallow-since")
-        [ "$mod" = "dnbd3" ]       && depth_arg+=("2019-02-12")
-        [ "$mod" = "rebash" ]      && depth_arg+=("2016-11-30")
-        [ "$mod" = "qemu-xmount" ] && depth_arg+=("2016-01-01")
-        [ "$mod" = "xmount" ]      && depth_arg+=("2015-11-05")
+        [ "$mod" = "dnbd3" ]                   && depth_arg+=("2019-02-12")
+        [ "$mod" = "rebash" ]                  && depth_arg+=("2016-11-30")
+        [ "$mod" = "qemu-xmount" ]             && depth_arg+=("2016-01-01")
+        [ "$mod" = "xmount" ]                  && depth_arg+=("2015-11-05")
+        [ "$mod" = "kernel-qcow2-util-linux" ] && depth_arg+=("2019-08-15")
         git clone -n --no-tags "${depth_arg[@]}" --branch "$branch" "$url" "$path"
         pushd "$path"
         git checkout "$commit"
@@ -62,11 +62,7 @@ if ! [[ -d "$_needed_location" ]]; then
         done 2>/dev/null
     '
     popd
-    # finally copy all dracut modules under modules.d/
-    cp --recursive \
-        "${_temporary_repository_location}/builder/modules.d/" \
-        "$(dirname $_needed_location)"
-    rm --recursive --force "$_temporary_repository_location"
+    ln -s "${_repo_dir}/builder/modules.d" "${_root_dir}/modules.d"
 fi
 set +o errexit
 ## endregion
@@ -374,6 +370,21 @@ parse_command_line() {
                 fi
                 shift
                 ;;
+            -q|--qcow-handler)
+                local given_argument="$1"
+                shift
+                qcow_handler="$1"
+                if [ -z "$qcow_handler" ]; then
+                    logging.critical \
+                        "Error with given option \"$given_argument\": This options needs to be either 'xmount' or 'kernel'."
+                    return 1
+                fi
+                shift
+                ;;
+            -u|--update)
+                shift
+                update='yes'
+                ;;
             -)
                 shift
                 while [[ "$1" =~ ^.+$ ]]; do
@@ -486,55 +497,7 @@ initialize_dracut() {
         "${_root_dir}/dracut/dracut-install"
     return $?
 }
-initialize_dnbd3_rootfs_module() {
-    # shellcheck disable=SC2016,SC2034
-    local __doc__='
-    Initialize dependency of dnbd3-rootfs modules:
-    xmount, qemu-xmount, dnbd3
-
-    Example:
-
-    `initialize_dnbd3_rootfs_module`
-    '
-    core.import "${_root_dir}/modules.d/dnbd3-rootfs/scripts/build.sh"
-    # We might want to move the "binaries" repos from the dnbd3-rootfs module to main repo one day...
-    _deps_base_dir="${_root_dir}/modules.d/dnbd3-rootfs/binaries"
-    if [[ ! -f "${_deps_base_dir}/xmount/trunk/build/src/xmount" ]]; then
-        logging.info "Could not find xmount binary, building it..."
-        if ! build_compile_xmount "${_deps_base_dir}/xmount/"; then
-            logging.error "Failed to build xmount binary."
-            return 1
-        fi
-    fi
-    if [[ ! -f "${_deps_base_dir}/qemu-xmount/libxmount_input_qemu.so" ]]; then
-        logging.info "Could not find xmount qemu library, building it..."
-        if ! build_compile_qemu_xmount "${_deps_base_dir}/qemu-xmount/"; then
-            logging error "Failed to build xmount qemu library."
-            return 1
-        fi
-    fi
-    # TODO check for its existence using modinfo -k <kernel_version>
-    if [[ ! -f "${_deps_base_dir}/dnbd3/build/dnbd3.ko" ]] || \
-        [[ ! -f "${_deps_base_dir}/dnbd3/build/dnbd3-client" ]]; then
-            logging.info "Could not find dnbd3, building it..."
-            if ! CMAKE_FLAGS="-DKERNEL_DIR=${kernel_headers}" \
-                build_compile_dnbd3 "${_deps_base_dir}/dnbd3/"; then
-                            logging.error "Failed to build dnbd3."
-                            return 1
-            fi
 
-    fi
-    if [[ ! -f "${_deps_base_dir}/systemd-preserve-process-marker/systemd-preserve-process-marker" ]]; then
-        logging.info "Could not find systemd-preserve-process-marker binary, building it ..."
-        if ! build_compile_systemd_preserve_process_marker \
-            "${_deps_base_dir}/systemd-preserve-process-marker/"; then
-                    logging.error "Failed to build systemd-preserve-process-marker"
-                    return 1
-        fi
-    fi
-    logging.info "Compilation of dnbd3-rootfs dependencies succeeded."
-    return 0
-}
 cleanup() {
     # shellcheck disable=SC2016,SC2034
     local __doc__='
@@ -567,11 +530,13 @@ main() {
     `main`
     '
     exceptions.activate
-    # region sanity check kernel version
+    # region sanity check kernel version and qcow handler
     # if no kernel was specified as dracut argument, use the running kernel's version
+    logging.info "Building for:"
     if [ -z "$kernel_version" ]; then
        kernel_version="$(uname -r)"
     fi
+    logging.info " * kernel version: $kernel_version"
     # similar for kernel headers needed to compile dnbd3 against.
     if [ -z "$kernel_headers" ]; then
        kernel_headers="/lib/modules/${kernel_version}/build"
@@ -579,7 +544,8 @@ main() {
     if [ ! -f "${kernel_headers}/Makefile" ]; then
        logging.critical 'Missing core dependency "linux-headers" for version to compile against given or current kernel.'
     fi
-    logging.info "Building for kernel version: ${kernel_version}."
+    logging.info " * kernel headers: $kernel_headers"
+    [ -n "$qcow_handler" ] && logging.info " * qcow2 handler: $qcow_handler"
     # endregion
     # region dependency checks and command line parsing
     result=0
@@ -666,6 +632,13 @@ main() {
         exit 0
     fi
     # endregion
+    # region handle '--update' to update all the modules in 'modules.d'
+    if [ "$update" == "yes" ]; then
+        pushd "${_repo_dir}"
+        git pull
+        popd
+    fi
+    # endregion
     # region handle dependencies which can be resolved automatically
     logging.info 'Checking dracut.'
     if [[ ! -f "${_root_dir}/dracut/dracut-install" ]]; then
@@ -719,10 +692,12 @@ main() {
     elif [[ "$cleanup" == 'yes' ]]; then
         logging.info 'Removing distribution specific files.'
         cleanup
-    elif [[ "$initialize" == 'yes' ]]; then
-        logging.info 'Prepare dnbd3-rootfs module dependencies.'
-        initialize_dnbd3_rootfs_module
     else
+        . "${_root_dir}/modules.d/dnbd3-rootfs/helper/build.inc"
+        initialize_components
+        if [[ "$initialize" == 'yes' ]]; then
+            exit 0
+        fi
         logging.info 'Build initramfs.'
         # NOTE: We deactivate our exception handle since dracut returns "1" if
         # it is launched with help parameter ("-h" or "--help").
diff --git a/builder/modules.d/dnbd3-rootfs/helper/build.inc b/builder/modules.d/dnbd3-rootfs/helper/build.inc
new file mode 100644
index 00000000..b7710985
--- /dev/null
+++ b/builder/modules.d/dnbd3-rootfs/helper/build.inc
@@ -0,0 +1,240 @@
+#!/usr/bin/env bash
+# -*- coding: utf-8 -*-
+# shellcheck source=./rebash/core.sh
+
+declare -rg _moddir="$(dirname "${BASH_SOURCE[0]}")/.."
+
+source "${_moddir}/scripts/rebash/core.sh"
+core.import logging
+
+declare -rg _supported_qcow_handlers=("xmount" "kernel")
+initialize_components() {
+    local qcow_handler="$1"
+    IFS='|' _pattern="^(${_supported_qcow_handlers[*]})$" export _pattern
+    if [[ ! "$qcow_handler" =~ $_pattern ]] ; then
+        logging.warn "Unknown qcow handler '$1' - will built all known."
+        logging.warn "Supported handlers: ${_supported_qcow_handlers[*]}"
+    fi
+    _deps_base_dir="${_moddir}/binaries"
+    # We might want to move the "binaries" repos from the dnbd3-rootfs module to main repo one day...
+    # TODO check for its existence using modinfo -k <kernel_version>
+    if [[ ! -f "${_deps_base_dir}/dnbd3/build/dnbd3.ko" ]] || \
+        [[ ! -f "${_deps_base_dir}/dnbd3/build/dnbd3-client" ]]; then
+            logging.info "Could not find dnbd3, building it..."
+            if ! CMAKE_FLAGS="-DKERNEL_DIR=${kernel_headers}" \
+                build_compile_dnbd3 "${_deps_base_dir}/dnbd3/"; then
+                            logging.error "Failed to build dnbd3."
+                            return 1
+            fi
+    fi
+    # take care of the qcow handler
+    if [ -z "$qcow_handler" ] || [ "$qcow_handler" = "xmount" ]; then
+        if [[ ! -f "${_deps_base_dir}/xmount/trunk/build/src/xmount" ]]; then
+            logging.info "Could not find xmount binary, building it..."
+            if ! build_compile_xmount "${_deps_base_dir}/xmount/"; then
+                logging.error "Failed to build xmount binary."
+                return 1
+            fi
+        fi
+        if [[ ! -f "${_deps_base_dir}/qemu-xmount/libxmount_input_qemu.so" ]]; then
+            logging.info "Could not find xmount qemu library, building it..."
+            if ! build_compile_qemu_xmount "${_deps_base_dir}/qemu-xmount/"; then
+                logging error "Failed to build xmount qemu library."
+                return 1
+            fi
+        fi
+    fi
+    if [ -z "$qcow_handler" ] || [ "$qcow_handler" = "kernel" ]; then
+        # TODO check if the given kernel supports it, bail otherwise
+        if [ ! -f "${_deps_base_dir}/kernel-qcow2-util-linux/losetup" ]; then
+            logging.info "Could not find losetup with qcow2 support, building it..."
+            if ! build_compile_losetup_qcow "${_deps_base_dir}/kernel-qcow2-util-linux"; then
+                logging.error "Failed to build losetup with qcow support."
+                return 1
+            fi
+        fi
+    fi
+
+    # always compile this helper since it does not cost much to do so
+    if [[ ! -f "${_deps_base_dir}/systemd-preserve-process-marker/systemd-preserve-process-marker" ]]; then
+        logging.info "Could not find systemd-preserve-process-marker binary, building it ..."
+        if ! build_compile_systemd_preserve_process_marker \
+            "${_deps_base_dir}/systemd-preserve-process-marker/"; then
+                    logging.error "Failed to build systemd-preserve-process-marker"
+                    return 1
+        fi
+    fi
+    logging.info "Compilation of dnbd3-rootfs dependencies succeeded."
+    return 0
+}
+
+clean_components() {
+    local __doc__='
+    Removes all compiled kernel specific files.
+    NOTE: This method is triggered manually and not supported by dracut itself.
+
+    Example:
+
+    `clean`
+    '
+    build_clean_xmount "${_moddir}/binaries/xmount/"
+    build_clean_qemu_xmount "${_moddir}/binaries/qemu-xmount/"
+    build_clean_dnbd3 "${_moddir}/binaries/dnbd3/"
+    build_clean_losetup_qcow "${_moddir}/binaries/kernel-qcow2-util-linux"
+    build_clean_dnbd3 "${_moddir}/binaries/dnbd3/"
+    build_clean_systemd_preserve_process_marker \
+        "${_moddir}/binaries/systemd-preserve-process-marker/"
+    return 0
+}
+# endregion
+
+build_compile_qemu_xmount() {
+    local __doc__='
+    Compiles qemu libxmount.
+    NOTE: expects xmount installation under
+    $1/../xmount/trunk/build/release_build/
+
+    Provides the following file:
+    "$1/libxmount_input_qemu.so"
+
+    Example:
+
+    `build_compile_qemu_xmount /qemu_source /xmount/installation`
+    '
+    pushd "$1"
+    local xmount_installation="../xmount/trunk/build/release_build/usr"
+    [ ! -z "$2" ] && xmount_installation="$2"
+    ./configure --enable-xmount-input --python="$(which python2)" \
+        --extra-cflags="-fPIC" \
+        --extra-cflags="-std=gnu99" \
+        --extra-cflags="-I${xmount_installation}/include" \
+        --extra-cflags="-I${xmount_installation}/include/xmount" \
+        --disable-fdt --target-list=""
+    make -j libxmount_input_qemu.so
+    local ret=$?
+    popd
+    return $ret
+}
+build_clean_qemu_xmount() {
+    local __doc__='Clean the build of `build_compile_qemu_xmount`.'
+    pushd "$1"
+    make clean
+    local ret=$?
+    popd
+    return $ret
+}
+build_compile_xmount() {
+    local __doc__='
+    Compiles xmount.
+
+    Provides the xmount installation under:
+    "$1/trunk/build/release_build/"
+
+    Example:
+
+    `build_compile_xmount /xmount_source /xmount_source/build /usr`
+    '
+    pushd "$1"
+
+    local destination_directory="./release_build"
+    [ ! -z "$2" ] && destination_directory="$2"
+    local install_prefix="/usr"
+    [ ! -z "$3" ] && install_prefix="$3"
+
+    mkdir --parents trunk/build
+    cd trunk/build || return 1
+    cmake -DCMAKE_BUILD_TYPE=Release \
+        -DCMAKE_INSTALL_PREFIX="$install_prefix" ..
+    make -j
+    make install DESTDIR="$destination_directory"
+    local ret=$?
+    popd
+    return $ret
+}
+build_clean_xmount() {
+    local __doc__='Clean the build of `build_compile_xmount`.'
+    rm --recursive --force "$1/trunk/build"
+}
+build_compile_dnbd3() {
+    local __doc__='
+    Compiles dnbd3 kernel module and client.
+
+    Provides the following file:
+    "$1/build/dnbd3.ko"
+    "$1/build/dnbd3-client"
+
+    Examples:
+
+    `build_compile_dnbd3 path/to/dnbd3/source/`
+
+    Passing the kernel version to cmake:
+    `CMAKE_FLAGS="-DKERNEL_VERSION=4.14.6-openslx+" \
+        build_compile_dnbd3 path/to/dnbd3/source/`
+    '
+    pushd "$1"
+    # NOTE: The generic way would be: "./build.sh" but this tries to build
+    # more than we really need and takes more time.
+    mkdir --parents build
+    pushd build
+    # Inject CMAKE_FLAGS as a way to control how cmake is called,
+    # e.g. to pass the kernel version
+    cmake ${CMAKE_FLAGS} ../
+    make -j dnbd3 dnbd3-client
+    local ret=$?
+    popd
+    return $ret
+}
+build_clean_dnbd3() {
+    local __doc__='Clean the build of `build_compile_dnbd3`.'
+    rm --recursive --force "$1/build"
+    return $?
+}
+# needs autopoint bison automake libtool
+build_compile_losetup_qcow() {
+    pushd "$1"
+    (
+        set -o errexit
+        ./autogen.sh
+        ./configure
+        make -j losetup
+    )
+    local ret=$?
+    popd
+    return $ret
+}
+build_clean_losetup_qcow() {
+    pushd "$1"
+    make clean
+    local ret=$?
+    popd
+    return $ret
+}
+build_compile_systemd_preserve_process_marker() {
+    local __doc__='
+    Compiles simple c program.
+
+    Examples:
+
+    `build_compile_systemd_preserve_process_marker path/to/program/folder`
+    '
+    pushd "$1"
+    make
+    local ret=$?
+    popd
+    return $ret
+}
+build_clean_systemd_preserve_process_marker() {
+    local __doc__='
+    Clean the build of
+    `build_compile_systemd_preserve_process_marker`.
+    '
+    pushd "$1"
+    make clean
+    local ret=$?
+    popd
+    return $ret
+}
+# region vim modline
+# vim: set tabstop=4 shiftwidth=4 expandtab:
+# vim: foldmethod=marker foldmarker=region,endregion:
+# endregion
diff --git a/builder/modules.d/dnbd3-rootfs/module-setup.sh b/builder/modules.d/dnbd3-rootfs/module-setup.sh
index 046732df..4e5f037f 100755
--- a/builder/modules.d/dnbd3-rootfs/module-setup.sh
+++ b/builder/modules.d/dnbd3-rootfs/module-setup.sh
@@ -5,7 +5,7 @@ source "$(dirname "${BASH_SOURCE[0]}")/scripts/rebash/core.sh"
 core.import exceptions
 core.import logging
 core.import utils
-core.import "$(core_abs_path "$(dirname "${BASH_SOURCE[0]}")/scripts/build.sh")"
+core.import "$(core_abs_path "$(dirname "${BASH_SOURCE[0]}")/helper/build.inc")"
 # endregion
 # region forward "build-initrfams.sh" logging configuration if present.
 _parse_dracut_args() {
@@ -68,6 +68,8 @@ _parse_dracut_args() {
 _debug=0
 _parse_dracut_args ${dracut_args[*]} || _debug=$?
 # endregion
+# region initialize function to build required components
+
 clean() {
     local __doc__='
     Removes all compiled kernel specific files.
@@ -77,13 +79,11 @@ clean() {
 
     `clean`
     '
-    build_clean_xmount "$moddir/binaries/xmount/"
-    build_clean_qemu_xmount "$moddir/binaries/qemu-xmount/"
-    build_clean_dnbd3 "$moddir/binaries/dnbd3/"
-    build_clean_systemd_preserve_process_marker \
-        "$moddir/binaries/systemd-preserve-process-marker/"
+    . "${moddir}/helper/build.inc"
+    clean_components
     return 0
 }
+# endregion
 # region dracut plugin api
 check() {
     local __doc__='
@@ -94,21 +94,15 @@ check() {
     `check`
     '
     exceptions.activate
+    initialize_components
     # NOTE: xmount must be compiled before qemu_xmount
     local xmount_is_built=true
-    if [[ ! -f "$moddir/binaries/xmount/trunk/build/src/xmount" ]]; then
-        if ! build_compile_xmount "$moddir/binaries/xmount/"; then
+    if [[ ! -f "$moddir/binaries/xmount/trunk/build/src/xmount" ]] ||
+        [[ -f "$moddir/binaries/qemu-xmount/libxmount_input_qemu.so" ]]; then
             xmount_is_built=false
-        fi
-    fi
-    if $xmount_is_built && [[ \
-        ! -f "$moddir/binaries/qemu-xmount/libxmount_input_qemu.so" \
-    ]]; then
-        build_compile_qemu_xmount "$moddir/binaries/qemu-xmount/" || \
-        xmount_is_built=false
     fi
     $xmount_is_built || logging.warn \
-        "Compiling \"xmount\" failed -> No support for container files (only raw images)."
+        "Compiling \"xmount\" failed."
 
     if [[ ! -f "$moddir/binaries/dnbd3/build/dnbd3.ko" ]] || \
        [[ ! -f "$moddir/binaries/dnbd3/build/dnbd3-client" ]]
@@ -153,7 +147,7 @@ depends() {
     '
     #local network_module="network"
     #dracut_module_included "systemd-initrd" && network_module="systemd-networkd-ext"
-    echo base bash kernel-modules shutdown slx-dmsetup slx-tools slx-network # "$network_module"
+    echo base bash kernel-modules shutdown slx-dmsetup slx-network # "$network_module"
 }
 installkernel() {
     local __doc__='
@@ -164,6 +158,7 @@ installkernel() {
 
     `installkernel`
     '
+    # TODO kqcow2 stuff
     inst "$moddir/binaries/dnbd3/build/dnbd3.ko" \
         /usr/lib/modules/current/extra/dnbd3.ko
 }
@@ -183,16 +178,27 @@ install() {
     # xmount
     local \
         xmount_installation="$moddir/binaries/xmount/trunk/build/release_build"
-    inst "${xmount_installation}/usr/bin/xmount" /usr/bin/xmount
-    for file in ${xmount_installation}/usr/lib/xmount/*; do
-        inst "$file" /usr/lib/xmount/"$(basename "$file")"
-    done
-    inst "$moddir/binaries/qemu-xmount/libxmount_input_qemu.so" \
-        /usr/lib/xmount/libxmount_input_qemu.so
-    # HACK fix missing libraries for some xmount input libs
-    # TODO copy the dependencies using dracut functions...
-    inst_libdir_file 'libafflib.so*'
-    inst_libdir_file 'libewf.so*'
+    if [[ -f "${xmount_installation}/usr/bin/xmount" ]]; then
+        inst "${xmount_installation}/usr/bin/xmount" /usr/bin/xmount
+        for file in ${xmount_installation}/usr/lib/xmount/*; do
+            inst "$file" /usr/lib/xmount/"$(basename "$file")"
+        done
+        inst "$moddir/binaries/qemu-xmount/libxmount_input_qemu.so" \
+            /usr/lib/xmount/libxmount_input_qemu.so
+        # HACK fix missing libraries for some xmount input libs
+        # TODO copy the dependencies using dracut functions...
+        inst_libdir_file 'libafflib.so*'
+        inst_libdir_file 'libewf.so*'
+    fi
+    # kqcow2
+    local kernel_qcow2_repo="$moddir/binaries/kernel-qcow2-util-linux"
+    if [[ -f "${kernel_qcow2_repo}/.libs/losetup" ]]; then
+        inst "${kernel_qcow2_repo}/.libs/losetup" /usr/local/bin/losetup
+        for lib in "${kernel_qcow2_repo}/"*.so*; do
+            # TODO check inst_libdir_file
+            inst "$file" "/usr/lib/$(basename $file)"
+        done
+    fi
     # endregion
     # region hooks
     inst_hook cmdline 00 "$moddir/hooks/enable-sysrq.sh"