[dnbd3-rootfs] various improvements

* use slx-tools to download the config
* use slx-network module instead of systemd-networkd
* remove container_unpack_xmount scripts and make it a function
* and more small things I can't think of ... :)

5 files changed, 84 insertions, 71 deletions

diff --git a/builder/modules.d/dnbd3-rootfs/hooks/copy-openslx-configuration-into-newroot.sh b/builder/modules.d/dnbd3-rootfs/hooks/copy-openslx-configuration-into-newroot.sh
index c827ec8d..12dab27d 100755
--- a/builder/modules.d/dnbd3-rootfs/hooks/copy-openslx-configuration-into-newroot.sh
+++ b/builder/modules.d/dnbd3-rootfs/hooks/copy-openslx-configuration-into-newroot.sh
@@ -18,7 +18,7 @@ mkdir --parents "${NEWROOT}/opt/openslx"
 cp "/etc/openslx" "${NEWROOT}/opt/openslx/config"
 
 echo "## Generated by '$0' in stage3" >> "${NEWROOT}/opt/openslx/config"
-cat "/run/openslx/pxe-network.conf" >> "${NEWROOT}/opt/openslx/config"
+cat "/run/openslx/network.conf" >> "${NEWROOT}/opt/openslx/config"
 
 # Set root/demo password for stage4, if set
 if [ -n "${SLX_ROOT_PASS}" ]; then
diff --git a/builder/modules.d/dnbd3-rootfs/hooks/fetch-config.sh b/builder/modules.d/dnbd3-rootfs/hooks/fetch-config.sh
index 212bda75..5096be9b 100755
--- a/builder/modules.d/dnbd3-rootfs/hooks/fetch-config.sh
+++ b/builder/modules.d/dnbd3-rootfs/hooks/fetch-config.sh
@@ -6,55 +6,54 @@ core.import exceptions
 core.import logging
 type emergency_shell >/dev/null 2>&1 || source /lib/dracut-lib.sh
 # endregion
+
 exceptions.try
 {
-[[ "$SLX_LOG_FILE_PATH" == "" ]] && SLX_LOG_FILE_PATH=/var/log/openslx
-logging.set_log_file "$SLX_LOG_FILE_PATH"
+logging.set_log_file "${SLX_LOG_FILE_PATH:-/var/log/openslx}"
 
 # NOTE: "getarg" raises an exception so deactivate exceptions for now.
 exceptions.deactivate
-configuration_file_name="$(getarg slx_configuration_filename=)"
-if [ -z "$configuration_file_name" ]; then
-	configuration_file_name='config'
-fi
 slx_server="$(getarg slxsrv=)"
 slx_server_base="$(getarg slxbase=)"
 exceptions.activate
 
+# build config_url
+config_url="http://${slx_server#@}/${slx_server_base}/config"
+
 # check if system's uuid was set
 if [ -s "/run/system-uuid" ]; then
 	uuid=$(cat "/run/system-uuid")
 	if [ -n "$uuid" ]; then
-		uuid_url="?uuid=$uuid"
+		config_url="${config_url}?uuid=${uuid}"
 	fi
 fi
 
-logging.info "Download SLX configuration from: \"$slx_server\"."
-(
-IFS=','
-for TIMEOUT in 1 1 2 3; do
-	if wget -q -t 2 -T $TIMEOUT \
-		"http://${slx_server#@}/${slx_server_base}/${configuration_file_name}${uuid_url}" \
-		--output-document '/etc/openslx.tmp'; then
-		break
-	fi
-done
-)
+config_path="/etc/openslx.tmp"
 
-if [[ ! -s "/etc/openslx.tmp" ]]; then
-	logging.warn "Downloading OpenSLX configuration file from any of the servers \"${slx_server}\" at location \"${slx_server_base}/${configuration_file_name}\" failed. Return code: $return_code"
-	return 1
+logging.info "Downloading '$config_url'..."
+slx-tools download_retry "$config_url" > "$config_path"
+
+if [ ! -s "$config_path" ] ; then
+    logging.warn "Downloading OpenSLX configuration file from '$config_url' failed with: $return_code"
+    emergency_shell "CRITICAL: System unusable."
+fi
+
+if ! ash -n "$config_path"; then
+    logging.warn "Downloaded OpenSLX configuration failed syntax check!"
+    emergency_shell "CRITICAL: System unusable."
 fi
 
 # remember kcl server and base
-(
+{
 	echo "SLX_KCL_SERVERS='$slx_server'"
 	echo "SLX_BASE_PATH='$slx_server_base'"
+    echo "# Config fetched from $config_url"
+    echo "CONFIG_DOWNLOAD_TIME=$(sed -r 's/^([0-9]+)\.([0-9]+).*$/\1\2/' /proc/uptime)"
 	echo '#_RCONFIG_TAG'
-) > /etc/openslx
+} > /etc/openslx
 
 # finally copy remote config into it
-cat /etc/openslx.tmp >> /etc/openslx
+cat "$config_path" >> /etc/openslx
 
 # slxsrv overrides SLX_DNBD3_SERVERS if prefixed with @
 [ "${slx_server#@}" != "${slx_server}" ] && sed -i "s/^SLX_DNBD3_SERVERS=.*/SLX_DNBD3_SERVERS='${slx_server#@}'/" "/etc/openslx"
diff --git a/builder/modules.d/dnbd3-rootfs/hooks/prepare-root-partition.sh b/builder/modules.d/dnbd3-rootfs/hooks/prepare-root-partition.sh
index 2d9e059c..ca572458 100755
--- a/builder/modules.d/dnbd3-rootfs/hooks/prepare-root-partition.sh
+++ b/builder/modules.d/dnbd3-rootfs/hooks/prepare-root-partition.sh
@@ -1,49 +1,64 @@
 #!/usr/bin/env bash
 # region imports
+type emergency_shell > /dev/null 2>&1 || source /lib/dracut-lib.sh
+source /etc/openslx
 source '/usr/lib/rebash/core.sh'
-core.import '/usr/lib/openslx/tools.sh'
 core.import exceptions
-core.import utils
 core.import logging
-type emergency_shell >/dev/null 2>&1 || source /lib/dracut-lib.sh
 # endregion
+
+# region globals/helper
+logging.set_log_file "${SLX_LOG_FILE_PATH:-/var/log/openslx}"
+# hardcode dnbd device path
+declare -rg _dnbd3_dev="/dev/dnbd0"
+
+# all outputs are redirected to stderr, since this functions should
+# only echo the path to the unpacked container to stdout.
+container_unpack_xmount() {
+    local in_device="$1"
+    local out_path="/mnt/xmount"
+    mkdir -p "$out_path"
+    # check tools first
+    if ! hash xmount systemd-preserve-process-marker; then
+        logging.warn "Missing xmount deps, will try raw..." 1>&2
+    elif ! systemd-preserve-process-marker xmount \
+            --in qemu "$in_device" \
+            --out raw "$out_path" &>/dev/null; then
+        logging.warn "xmount call failed, assuming raw image." 1>&2
+    else
+        in_device="${out_path}/${_dnbd3_dev##*/}.dd"
+    fi
+    local loop_device="$(losetup -f)"
+    losetup "$loop_device" "$in_device" --partscan
+    udevadm settle
+    echo "$loop_device"
+}
+# endregion
+
 exceptions.try
 {
-source /etc/openslx
-logging.set_commands_level debug
-logging.set_level debug
-[[ "$SLX_LOG_FILE_PATH" == "" ]] && SLX_LOG_FILE_PATH=/var/log/openslx
-logging.set_log_file "$SLX_LOG_FILE_PATH"
-
 # region connect dnbd3 image
-IFS_backup="$IFS"
+(
 IFS=", "
-return_code=1
-for host in ${SLX_DNBD3_SERVERS}; do
+for host in ${SLX_DNBD3_SERVERS} FAIL; do
+    if [ "$host" = "FAIL" ]; then
+        emergency_shell "Failed to connect '${SLX_DNBD3_IMAGE}' "\
+            "${SLX_DNBD3_RID:+(revision: $SLX_DNBD3_RID)} "
+            "from one of '$SLX_DNBD3_SERVERS' to '$_dnbd3_dev'."
+    fi
     logging.info "Trying host \"$host\"."
-    if systemd-preserve-process-marker dnbd3-client --host "$host" --image \
-        "${SLX_DNBD3_IMAGE}" --device "$SLX_DNBD3_DEVICE" \
-        --rid "$SLX_DNBD3_RID"; then
-        return_code=0
+    if systemd-preserve-process-marker dnbd3-client \
+        --host "$host" \
+        --image "${SLX_DNBD3_IMAGE}" \
+        --device "$_dnbd3_dev" \
+        ${SLX_DNBD3_RID=+--rid "$SLX_DNBD3_RID"}; then
         break
     fi
 done
-IFS="$IFS_backup"
-
-if [[ $return_code != 0 ]]; then
-    logging.warn "Failed to connect \"${SLX_DNBD3_IMAGE}\" (revision" \
-        "\"$SLX_DNBD3_RID\") from one of \"$SLX_DNBD3_SERVERS\" to" \
-        "\"$SLX_DNBD3_DEVICE\"."
-    exit 1
-fi
+)
 # endregion
 # region unpack dnbd3 image with xmount
-if [ "$SLX_LOG_FILE_PATH" != "" ]; then
-    read_only_device="$(container-unpack-xmount "$SLX_DNBD3_DEVICE" \
-        2>>"$SLX_LOG_FILE_PATH")"
-else
-    read_only_device="$(container-unpack-xmount "$SLX_DNBD3_DEVICE")"
-fi
+read_only_device="$(container_unpack_xmount "$_dnbd3_dev")"
 
 # Fail fast if unpacking dnbd3 image failed.
 [ -z "$read_only_device" ] && exit 1
@@ -56,7 +71,7 @@ if [ -z "$SLX_SYSTEM_PARTITION_PREPARATION_SCRIPT" ]; then
         read_only_partition="$read_only_device"
         true
     else
-        read_only_partition="$(get-partitions-by-id \
+        read_only_partition="$(slx-tools dev_find_partitions \
             "$read_only_device" "$SLX_SYSTEM_PARTITION_IDENTIFIER")"
     fi
 else
@@ -69,11 +84,11 @@ if [[ ! $? || -z "$read_only_partition" ]]; then
     exit 1
 fi
 logging.info "Using read-only partition: $read_only_partition"
-
 # endregion
 
-# region RW layer through slx-partitioner
-dmsetup-slx-device $read_only_partition
+# region add rw layer to dnbd3 image
+# don't be fooled to think we are done, the next part is crucial
+dmsetup-slx-device "$read_only_partition"
 # endregion
 
 }
diff --git a/builder/modules.d/dnbd3-rootfs/module-setup.sh b/builder/modules.d/dnbd3-rootfs/module-setup.sh
index 26678194..d793be44 100755
--- a/builder/modules.d/dnbd3-rootfs/module-setup.sh
+++ b/builder/modules.d/dnbd3-rootfs/module-setup.sh
@@ -151,9 +151,9 @@ depends() {
     +doc_test_contains
     base
     '
-    local network_module="network"
-    dracut_module_included "systemd-initrd" && network_module="systemd-networkd-ext"
-    echo base bash kernel-modules shutdown slx-partitioner "$network_module"
+    #local network_module="network"
+    #dracut_module_included "systemd-initrd" && network_module="systemd-networkd-ext"
+    echo base bash kernel-modules shutdown slx-partitioner slx-tools slx-network # "$network_module"
 }
 installkernel() {
     local __doc__='
@@ -183,10 +183,10 @@ install() {
     ## DEPRECATED
     #inst "$moddir/scripts/device-add-write-layer.sh" \
     #    /usr/bin/device-add-write-layer
-    inst "$moddir/scripts/container-unpack-xmount.sh" \
-        /usr/bin/container-unpack-xmount
-    inst "$moddir/scripts/container-unpack-qemu.sh" \
-        /usr/bin/container-unpack-qemu
+    #inst "$moddir/scripts/container-unpack-xmount.sh" \
+    #    /usr/bin/container-unpack-xmount
+    #inst "$moddir/scripts/container-unpack-qemu.sh" \
+    #    /usr/bin/container-unpack-qemu
     # xmount
     local \
         xmount_installation="$moddir/binaries/xmount/trunk/build/release_build"
@@ -243,7 +243,7 @@ install() {
     for file_path in "$moddir/scripts/rebash/"*; do
         inst "$file_path" "/usr/lib/rebash/$(basename "$file_path")"
     done
-    inst "$moddir/scripts/tools.sh" "/usr/lib/openslx/tools.sh"
+    #inst "$moddir/scripts/tools.sh" "/usr/lib/openslx/tools.sh"
     # endregion
     # region configuration files
     # Use terminal readline settings from the template system.
@@ -257,7 +257,7 @@ install() {
     inst_multiple \
         awk \
         basename bash blockdev \
-        cat cut \
+        cat cut curl \
         dd diff dirname dmsetup \
         find \
         grep \
@@ -265,8 +265,7 @@ install() {
         losetup lsblk \
         mkfifo mktemp mount mountpoint \
         sed sleep sort \
-        tee touch tr \
-        wget
+        tee touch tr
 }
 # endregion
 # region vim modline
diff --git a/builder/modules.d/dnbd3-rootfs/scripts/build.sh b/builder/modules.d/dnbd3-rootfs/scripts/build.sh
index 16e8848d..4df163df 100644
--- a/builder/modules.d/dnbd3-rootfs/scripts/build.sh
+++ b/builder/modules.d/dnbd3-rootfs/scripts/build.sh
@@ -90,7 +90,7 @@ build_compile_dnbd3() {
     # NOTE: The generic way would be: "./build.sh" but this tries to build
     # more than we really need and takes more time.
     mkdir --parents build
-    cd build
+    pushd build
     # Inject CMAKE_FLAGS as a way to control how cmake is called,
     # e.g. to pass the kernel version
     cmake ${CMAKE_FLAGS} ../