From 53074916a2ccb2dad8ae1fc7eec7a112348b179e Mon Sep 17 00:00:00 2001 From: Jonathan Bauer Date: Tue, 3 Sep 2019 13:31:11 +0200 Subject: initial support for different qcow handlers --- builder/build-initramfs.sh | 113 +++++------ builder/modules.d/dnbd3-rootfs/helper/build.inc | 240 ++++++++++++++++++++++++ builder/modules.d/dnbd3-rootfs/module-setup.sh | 60 +++--- 3 files changed, 317 insertions(+), 96 deletions(-) create mode 100644 builder/modules.d/dnbd3-rootfs/helper/build.inc diff --git a/builder/build-initramfs.sh b/builder/build-initramfs.sh index 66dbf172..e10ac67c 100755 --- a/builder/build-initramfs.sh +++ b/builder/build-initramfs.sh @@ -22,21 +22,20 @@ # Note that you only get very necessary output until you provide "--verbose" as # commandline options. declare -rg _root_dir="$(readlink -f $(dirname ${BASH_SOURCE[0]}))" +declare -rg _repo_dir="${_root_dir}/systemd-init.git" declare -rg _git_source="git://git.openslx.org/openslx-ng/systemd-init.git" declare -rg _git_branch="master" ## region ensure presence of needed dependencies set -o errexit -_needed_location="${_root_dir}/modules.d" -if ! [[ -d "$_needed_location" ]]; then - echo "The dracut modules aren't available, loading them." +if [ ! -e "$_repo_dir" ]; then + echo "Missing dracut modules repository, loading them." if ! hash git; then echo "Needed dependency \"git\" isn't available." - echo "Please install \"git\" or provide the repositories data structure in \"${_root_dir}\"." + echo "Please install \"git\" or provide the main repository in \"${_repo_dir}\"." fi - _temporary_repository_location="$(mktemp --directory)" git clone --branch "$_git_branch" --single-branch --depth 1 \ - "$_git_source" "$_temporary_repository_location" - pushd "$_temporary_repository_location" + "$_git_source" "${_repo_dir}" + pushd "${_repo_dir}" git submodule init # try to clone submodules as shallowy as possible, since we cannot just # use '--depth 1' on submodules residing on non-master branches... @@ -46,10 +45,11 @@ if ! [[ -d "$_needed_location" ]]; then branch="$(git config -f .gitmodules --get submodule.${mod}.branch)" commit="$(git submodule status $path | grep -oE '[0-9a-f]{40}')" depth_arg=("--shallow-since") - [ "$mod" = "dnbd3" ] && depth_arg+=("2019-02-12") - [ "$mod" = "rebash" ] && depth_arg+=("2016-11-30") - [ "$mod" = "qemu-xmount" ] && depth_arg+=("2016-01-01") - [ "$mod" = "xmount" ] && depth_arg+=("2015-11-05") + [ "$mod" = "dnbd3" ] && depth_arg+=("2019-02-12") + [ "$mod" = "rebash" ] && depth_arg+=("2016-11-30") + [ "$mod" = "qemu-xmount" ] && depth_arg+=("2016-01-01") + [ "$mod" = "xmount" ] && depth_arg+=("2015-11-05") + [ "$mod" = "kernel-qcow2-util-linux" ] && depth_arg+=("2019-08-15") git clone -n --no-tags "${depth_arg[@]}" --branch "$branch" "$url" "$path" pushd "$path" git checkout "$commit" @@ -62,11 +62,7 @@ if ! [[ -d "$_needed_location" ]]; then done 2>/dev/null ' popd - # finally copy all dracut modules under modules.d/ - cp --recursive \ - "${_temporary_repository_location}/builder/modules.d/" \ - "$(dirname $_needed_location)" - rm --recursive --force "$_temporary_repository_location" + ln -s "${_repo_dir}/builder/modules.d" "${_root_dir}/modules.d" fi set +o errexit ## endregion @@ -374,6 +370,21 @@ parse_command_line() { fi shift ;; + -q|--qcow-handler) + local given_argument="$1" + shift + qcow_handler="$1" + if [ -z "$qcow_handler" ]; then + logging.critical \ + "Error with given option \"$given_argument\": This options needs to be either 'xmount' or 'kernel'." + return 1 + fi + shift + ;; + -u|--update) + shift + update='yes' + ;; -) shift while [[ "$1" =~ ^.+$ ]]; do @@ -486,55 +497,7 @@ initialize_dracut() { "${_root_dir}/dracut/dracut-install" return $? } -initialize_dnbd3_rootfs_module() { - # shellcheck disable=SC2016,SC2034 - local __doc__=' - Initialize dependency of dnbd3-rootfs modules: - xmount, qemu-xmount, dnbd3 - - Example: - - `initialize_dnbd3_rootfs_module` - ' - core.import "${_root_dir}/modules.d/dnbd3-rootfs/scripts/build.sh" - # We might want to move the "binaries" repos from the dnbd3-rootfs module to main repo one day... - _deps_base_dir="${_root_dir}/modules.d/dnbd3-rootfs/binaries" - if [[ ! -f "${_deps_base_dir}/xmount/trunk/build/src/xmount" ]]; then - logging.info "Could not find xmount binary, building it..." - if ! build_compile_xmount "${_deps_base_dir}/xmount/"; then - logging.error "Failed to build xmount binary." - return 1 - fi - fi - if [[ ! -f "${_deps_base_dir}/qemu-xmount/libxmount_input_qemu.so" ]]; then - logging.info "Could not find xmount qemu library, building it..." - if ! build_compile_qemu_xmount "${_deps_base_dir}/qemu-xmount/"; then - logging error "Failed to build xmount qemu library." - return 1 - fi - fi - # TODO check for its existence using modinfo -k - if [[ ! -f "${_deps_base_dir}/dnbd3/build/dnbd3.ko" ]] || \ - [[ ! -f "${_deps_base_dir}/dnbd3/build/dnbd3-client" ]]; then - logging.info "Could not find dnbd3, building it..." - if ! CMAKE_FLAGS="-DKERNEL_DIR=${kernel_headers}" \ - build_compile_dnbd3 "${_deps_base_dir}/dnbd3/"; then - logging.error "Failed to build dnbd3." - return 1 - fi - fi - if [[ ! -f "${_deps_base_dir}/systemd-preserve-process-marker/systemd-preserve-process-marker" ]]; then - logging.info "Could not find systemd-preserve-process-marker binary, building it ..." - if ! build_compile_systemd_preserve_process_marker \ - "${_deps_base_dir}/systemd-preserve-process-marker/"; then - logging.error "Failed to build systemd-preserve-process-marker" - return 1 - fi - fi - logging.info "Compilation of dnbd3-rootfs dependencies succeeded." - return 0 -} cleanup() { # shellcheck disable=SC2016,SC2034 local __doc__=' @@ -567,11 +530,13 @@ main() { `main` ' exceptions.activate - # region sanity check kernel version + # region sanity check kernel version and qcow handler # if no kernel was specified as dracut argument, use the running kernel's version + logging.info "Building for:" if [ -z "$kernel_version" ]; then kernel_version="$(uname -r)" fi + logging.info " * kernel version: $kernel_version" # similar for kernel headers needed to compile dnbd3 against. if [ -z "$kernel_headers" ]; then kernel_headers="/lib/modules/${kernel_version}/build" @@ -579,7 +544,8 @@ main() { if [ ! -f "${kernel_headers}/Makefile" ]; then logging.critical 'Missing core dependency "linux-headers" for version to compile against given or current kernel.' fi - logging.info "Building for kernel version: ${kernel_version}." + logging.info " * kernel headers: $kernel_headers" + [ -n "$qcow_handler" ] && logging.info " * qcow2 handler: $qcow_handler" # endregion # region dependency checks and command line parsing result=0 @@ -666,6 +632,13 @@ main() { exit 0 fi # endregion + # region handle '--update' to update all the modules in 'modules.d' + if [ "$update" == "yes" ]; then + pushd "${_repo_dir}" + git pull + popd + fi + # endregion # region handle dependencies which can be resolved automatically logging.info 'Checking dracut.' if [[ ! -f "${_root_dir}/dracut/dracut-install" ]]; then @@ -719,10 +692,12 @@ main() { elif [[ "$cleanup" == 'yes' ]]; then logging.info 'Removing distribution specific files.' cleanup - elif [[ "$initialize" == 'yes' ]]; then - logging.info 'Prepare dnbd3-rootfs module dependencies.' - initialize_dnbd3_rootfs_module else + . "${_root_dir}/modules.d/dnbd3-rootfs/helper/build.inc" + initialize_components + if [[ "$initialize" == 'yes' ]]; then + exit 0 + fi logging.info 'Build initramfs.' # NOTE: We deactivate our exception handle since dracut returns "1" if # it is launched with help parameter ("-h" or "--help"). diff --git a/builder/modules.d/dnbd3-rootfs/helper/build.inc b/builder/modules.d/dnbd3-rootfs/helper/build.inc new file mode 100644 index 00000000..b7710985 --- /dev/null +++ b/builder/modules.d/dnbd3-rootfs/helper/build.inc @@ -0,0 +1,240 @@ +#!/usr/bin/env bash +# -*- coding: utf-8 -*- +# shellcheck source=./rebash/core.sh + +declare -rg _moddir="$(dirname "${BASH_SOURCE[0]}")/.." + +source "${_moddir}/scripts/rebash/core.sh" +core.import logging + +declare -rg _supported_qcow_handlers=("xmount" "kernel") +initialize_components() { + local qcow_handler="$1" + IFS='|' _pattern="^(${_supported_qcow_handlers[*]})$" export _pattern + if [[ ! "$qcow_handler" =~ $_pattern ]] ; then + logging.warn "Unknown qcow handler '$1' - will built all known." + logging.warn "Supported handlers: ${_supported_qcow_handlers[*]}" + fi + _deps_base_dir="${_moddir}/binaries" + # We might want to move the "binaries" repos from the dnbd3-rootfs module to main repo one day... + # TODO check for its existence using modinfo -k + if [[ ! -f "${_deps_base_dir}/dnbd3/build/dnbd3.ko" ]] || \ + [[ ! -f "${_deps_base_dir}/dnbd3/build/dnbd3-client" ]]; then + logging.info "Could not find dnbd3, building it..." + if ! CMAKE_FLAGS="-DKERNEL_DIR=${kernel_headers}" \ + build_compile_dnbd3 "${_deps_base_dir}/dnbd3/"; then + logging.error "Failed to build dnbd3." + return 1 + fi + fi + # take care of the qcow handler + if [ -z "$qcow_handler" ] || [ "$qcow_handler" = "xmount" ]; then + if [[ ! -f "${_deps_base_dir}/xmount/trunk/build/src/xmount" ]]; then + logging.info "Could not find xmount binary, building it..." + if ! build_compile_xmount "${_deps_base_dir}/xmount/"; then + logging.error "Failed to build xmount binary." + return 1 + fi + fi + if [[ ! -f "${_deps_base_dir}/qemu-xmount/libxmount_input_qemu.so" ]]; then + logging.info "Could not find xmount qemu library, building it..." + if ! build_compile_qemu_xmount "${_deps_base_dir}/qemu-xmount/"; then + logging error "Failed to build xmount qemu library." + return 1 + fi + fi + fi + if [ -z "$qcow_handler" ] || [ "$qcow_handler" = "kernel" ]; then + # TODO check if the given kernel supports it, bail otherwise + if [ ! -f "${_deps_base_dir}/kernel-qcow2-util-linux/losetup" ]; then + logging.info "Could not find losetup with qcow2 support, building it..." + if ! build_compile_losetup_qcow "${_deps_base_dir}/kernel-qcow2-util-linux"; then + logging.error "Failed to build losetup with qcow support." + return 1 + fi + fi + fi + + # always compile this helper since it does not cost much to do so + if [[ ! -f "${_deps_base_dir}/systemd-preserve-process-marker/systemd-preserve-process-marker" ]]; then + logging.info "Could not find systemd-preserve-process-marker binary, building it ..." + if ! build_compile_systemd_preserve_process_marker \ + "${_deps_base_dir}/systemd-preserve-process-marker/"; then + logging.error "Failed to build systemd-preserve-process-marker" + return 1 + fi + fi + logging.info "Compilation of dnbd3-rootfs dependencies succeeded." + return 0 +} + +clean_components() { + local __doc__=' + Removes all compiled kernel specific files. + NOTE: This method is triggered manually and not supported by dracut itself. + + Example: + + `clean` + ' + build_clean_xmount "${_moddir}/binaries/xmount/" + build_clean_qemu_xmount "${_moddir}/binaries/qemu-xmount/" + build_clean_dnbd3 "${_moddir}/binaries/dnbd3/" + build_clean_losetup_qcow "${_moddir}/binaries/kernel-qcow2-util-linux" + build_clean_dnbd3 "${_moddir}/binaries/dnbd3/" + build_clean_systemd_preserve_process_marker \ + "${_moddir}/binaries/systemd-preserve-process-marker/" + return 0 +} +# endregion + +build_compile_qemu_xmount() { + local __doc__=' + Compiles qemu libxmount. + NOTE: expects xmount installation under + $1/../xmount/trunk/build/release_build/ + + Provides the following file: + "$1/libxmount_input_qemu.so" + + Example: + + `build_compile_qemu_xmount /qemu_source /xmount/installation` + ' + pushd "$1" + local xmount_installation="../xmount/trunk/build/release_build/usr" + [ ! -z "$2" ] && xmount_installation="$2" + ./configure --enable-xmount-input --python="$(which python2)" \ + --extra-cflags="-fPIC" \ + --extra-cflags="-std=gnu99" \ + --extra-cflags="-I${xmount_installation}/include" \ + --extra-cflags="-I${xmount_installation}/include/xmount" \ + --disable-fdt --target-list="" + make -j libxmount_input_qemu.so + local ret=$? + popd + return $ret +} +build_clean_qemu_xmount() { + local __doc__='Clean the build of `build_compile_qemu_xmount`.' + pushd "$1" + make clean + local ret=$? + popd + return $ret +} +build_compile_xmount() { + local __doc__=' + Compiles xmount. + + Provides the xmount installation under: + "$1/trunk/build/release_build/" + + Example: + + `build_compile_xmount /xmount_source /xmount_source/build /usr` + ' + pushd "$1" + + local destination_directory="./release_build" + [ ! -z "$2" ] && destination_directory="$2" + local install_prefix="/usr" + [ ! -z "$3" ] && install_prefix="$3" + + mkdir --parents trunk/build + cd trunk/build || return 1 + cmake -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="$install_prefix" .. + make -j + make install DESTDIR="$destination_directory" + local ret=$? + popd + return $ret +} +build_clean_xmount() { + local __doc__='Clean the build of `build_compile_xmount`.' + rm --recursive --force "$1/trunk/build" +} +build_compile_dnbd3() { + local __doc__=' + Compiles dnbd3 kernel module and client. + + Provides the following file: + "$1/build/dnbd3.ko" + "$1/build/dnbd3-client" + + Examples: + + `build_compile_dnbd3 path/to/dnbd3/source/` + + Passing the kernel version to cmake: + `CMAKE_FLAGS="-DKERNEL_VERSION=4.14.6-openslx+" \ + build_compile_dnbd3 path/to/dnbd3/source/` + ' + pushd "$1" + # NOTE: The generic way would be: "./build.sh" but this tries to build + # more than we really need and takes more time. + mkdir --parents build + pushd build + # Inject CMAKE_FLAGS as a way to control how cmake is called, + # e.g. to pass the kernel version + cmake ${CMAKE_FLAGS} ../ + make -j dnbd3 dnbd3-client + local ret=$? + popd + return $ret +} +build_clean_dnbd3() { + local __doc__='Clean the build of `build_compile_dnbd3`.' + rm --recursive --force "$1/build" + return $? +} +# needs autopoint bison automake libtool +build_compile_losetup_qcow() { + pushd "$1" + ( + set -o errexit + ./autogen.sh + ./configure + make -j losetup + ) + local ret=$? + popd + return $ret +} +build_clean_losetup_qcow() { + pushd "$1" + make clean + local ret=$? + popd + return $ret +} +build_compile_systemd_preserve_process_marker() { + local __doc__=' + Compiles simple c program. + + Examples: + + `build_compile_systemd_preserve_process_marker path/to/program/folder` + ' + pushd "$1" + make + local ret=$? + popd + return $ret +} +build_clean_systemd_preserve_process_marker() { + local __doc__=' + Clean the build of + `build_compile_systemd_preserve_process_marker`. + ' + pushd "$1" + make clean + local ret=$? + popd + return $ret +} +# region vim modline +# vim: set tabstop=4 shiftwidth=4 expandtab: +# vim: foldmethod=marker foldmarker=region,endregion: +# endregion diff --git a/builder/modules.d/dnbd3-rootfs/module-setup.sh b/builder/modules.d/dnbd3-rootfs/module-setup.sh index 046732df..4e5f037f 100755 --- a/builder/modules.d/dnbd3-rootfs/module-setup.sh +++ b/builder/modules.d/dnbd3-rootfs/module-setup.sh @@ -5,7 +5,7 @@ source "$(dirname "${BASH_SOURCE[0]}")/scripts/rebash/core.sh" core.import exceptions core.import logging core.import utils -core.import "$(core_abs_path "$(dirname "${BASH_SOURCE[0]}")/scripts/build.sh")" +core.import "$(core_abs_path "$(dirname "${BASH_SOURCE[0]}")/helper/build.inc")" # endregion # region forward "build-initrfams.sh" logging configuration if present. _parse_dracut_args() { @@ -68,6 +68,8 @@ _parse_dracut_args() { _debug=0 _parse_dracut_args ${dracut_args[*]} || _debug=$? # endregion +# region initialize function to build required components + clean() { local __doc__=' Removes all compiled kernel specific files. @@ -77,13 +79,11 @@ clean() { `clean` ' - build_clean_xmount "$moddir/binaries/xmount/" - build_clean_qemu_xmount "$moddir/binaries/qemu-xmount/" - build_clean_dnbd3 "$moddir/binaries/dnbd3/" - build_clean_systemd_preserve_process_marker \ - "$moddir/binaries/systemd-preserve-process-marker/" + . "${moddir}/helper/build.inc" + clean_components return 0 } +# endregion # region dracut plugin api check() { local __doc__=' @@ -94,21 +94,15 @@ check() { `check` ' exceptions.activate + initialize_components # NOTE: xmount must be compiled before qemu_xmount local xmount_is_built=true - if [[ ! -f "$moddir/binaries/xmount/trunk/build/src/xmount" ]]; then - if ! build_compile_xmount "$moddir/binaries/xmount/"; then + if [[ ! -f "$moddir/binaries/xmount/trunk/build/src/xmount" ]] || + [[ -f "$moddir/binaries/qemu-xmount/libxmount_input_qemu.so" ]]; then xmount_is_built=false - fi - fi - if $xmount_is_built && [[ \ - ! -f "$moddir/binaries/qemu-xmount/libxmount_input_qemu.so" \ - ]]; then - build_compile_qemu_xmount "$moddir/binaries/qemu-xmount/" || \ - xmount_is_built=false fi $xmount_is_built || logging.warn \ - "Compiling \"xmount\" failed -> No support for container files (only raw images)." + "Compiling \"xmount\" failed." if [[ ! -f "$moddir/binaries/dnbd3/build/dnbd3.ko" ]] || \ [[ ! -f "$moddir/binaries/dnbd3/build/dnbd3-client" ]] @@ -153,7 +147,7 @@ depends() { ' #local network_module="network" #dracut_module_included "systemd-initrd" && network_module="systemd-networkd-ext" - echo base bash kernel-modules shutdown slx-dmsetup slx-tools slx-network # "$network_module" + echo base bash kernel-modules shutdown slx-dmsetup slx-network # "$network_module" } installkernel() { local __doc__=' @@ -164,6 +158,7 @@ installkernel() { `installkernel` ' + # TODO kqcow2 stuff inst "$moddir/binaries/dnbd3/build/dnbd3.ko" \ /usr/lib/modules/current/extra/dnbd3.ko } @@ -183,16 +178,27 @@ install() { # xmount local \ xmount_installation="$moddir/binaries/xmount/trunk/build/release_build" - inst "${xmount_installation}/usr/bin/xmount" /usr/bin/xmount - for file in ${xmount_installation}/usr/lib/xmount/*; do - inst "$file" /usr/lib/xmount/"$(basename "$file")" - done - inst "$moddir/binaries/qemu-xmount/libxmount_input_qemu.so" \ - /usr/lib/xmount/libxmount_input_qemu.so - # HACK fix missing libraries for some xmount input libs - # TODO copy the dependencies using dracut functions... - inst_libdir_file 'libafflib.so*' - inst_libdir_file 'libewf.so*' + if [[ -f "${xmount_installation}/usr/bin/xmount" ]]; then + inst "${xmount_installation}/usr/bin/xmount" /usr/bin/xmount + for file in ${xmount_installation}/usr/lib/xmount/*; do + inst "$file" /usr/lib/xmount/"$(basename "$file")" + done + inst "$moddir/binaries/qemu-xmount/libxmount_input_qemu.so" \ + /usr/lib/xmount/libxmount_input_qemu.so + # HACK fix missing libraries for some xmount input libs + # TODO copy the dependencies using dracut functions... + inst_libdir_file 'libafflib.so*' + inst_libdir_file 'libewf.so*' + fi + # kqcow2 + local kernel_qcow2_repo="$moddir/binaries/kernel-qcow2-util-linux" + if [[ -f "${kernel_qcow2_repo}/.libs/losetup" ]]; then + inst "${kernel_qcow2_repo}/.libs/losetup" /usr/local/bin/losetup + for lib in "${kernel_qcow2_repo}/"*.so*; do + # TODO check inst_libdir_file + inst "$file" "/usr/lib/$(basename $file)" + done + fi # endregion # region hooks inst_hook cmdline 00 "$moddir/hooks/enable-sysrq.sh" -- cgit v1.2.3-55-g7522