path: root/core/bin/setup_target

#!/bin/bash
# -----------------------------------------------------------------------------
#
# Copyright (c) 2011..2018 bwLehrpool-Projektteam
#
# This program/file is free software distributed under the GPL version 2.
# See https://www.gnu.org/licenses/old-licenses/gpl-2.0-standalone.html
#
# If you have any feedback please consult https://bwlehrpool.de and
# send your feedback to support@bwlehrpool.de.
#
# General information about bwLehrpool can be found at https://bwlehrpool.de
#
# -----------------------------------------------------------------------------
#
#	This file contains the functions needed to setup a target.
#	The main function is 'generate_target' which will go over all
#	activated modules (symlinks found under core/targets/<target> or
#	overlay/targets/<target>) and run the module-specific 'fetch_source',
#	'build' and 'post_copy' functions of the modules.
#
#	The functions defined here are module-independent, meaning they
#	can process any module, as long as following requirements are met:
#
#		- Modules need to have a build-script and a config file,
#		  i.e. core/module/kernel/module.build
#		  and  core/module/kernel/module.conf
#		  as these will be read by 'process_module'.
#
#		- Modules do not change anything inside the module dirs anymore,
#		  everything (triggerfiles, source download, build) is done in
#		  tmp/work/<MODULE>
#
#		- Several information have to be set in the config file of
#		  a module:
#		    * REQUIRED_BINARIES - list of binaries
#		    * REQUIRED_FILES - list of regular files
#		    * REQUIRED_DIRECTORIES - list of directories
#		    * REQUIRED_SYSTEM_FILES - files to be copied directly from the system
#		    * REQUIRED_INSTALLED_PACKAGES - list of packages needed at build-time
#		    * REQUIRED_CONTENT_PACKAGES - list of packages where the contents are to be copied
#		    * (optional variables needed by the module-build-script)
#
#		  NOTE: REQUIRED_BINARIES, REQUIRED_FILES, REQUIRED_FILES will be ONLY looked
#		        for in the module's own build directory!
#
#	Optionally, a module can have a static data directory under
#	core/modules/<module>/data which will be copied as is to the target build directory.
#
#	Moreover modules do not need to specify the dynamic dependencies of given binaries
#	as these will be determined automaticly by the helper function 'get_dynamic_dependencies'
# -----------------------------------------------------------------------------
#
# Map modules from their names to path - contains all modules of current target
declare -Ag MODULE_PATHS=()
declare -g PACKAGES_INSTALLED_FLAG=
declare -rg CORE_DIR="${ROOT_DIR}/core"
declare -rg TARGETS_BASEDIR="$( readlink -f "${CORE_DIR}/targets" )"
declare -rg EXPORT_DIR="/export/build"
[ -z "$REMOTE_EXPORT_DIR" ] || EXPORT_DIR=$REMOTE_EXPORT_DIR

initial_checks () {
	get_kernel_version
	detect_distribution
}

unset_required_vars () {
	for VARNAME in ${!REQUIRED_*}; do
		unset "$VARNAME"
	done
}

# Install all the packages from all selected modules in current target
install_all_packages () {
	[ -n "$PACKAGES_INSTALLED_FLAG" ] && return 0
	PACKAGES_INSTALLED_FLAG=1
	local module file config
	declare -a missing
	declare -A packages dupcheck
	packages=()
	for module in $MODULES; do
		unset_required_vars
		config="${MODULE_PATHS["$module"]}/module.conf"
		for file in $SYS_VERSIONS; do
			if [ -e "${config}.${file}" ]; then
				validate_config "${config}.${file}"
				. "${config}.${file}"
				break
			fi
		done
		dupcheck=()
		for file in $REQUIRED_INSTALLED_PACKAGES; do
			[ -n "${dupcheck["$file"]}" ] && continue
			packages["$file"]+=" $module"
			dupcheck["$file"]=1
		done
		for file in $REQUIRED_CONTENT_PACKAGES; do
			[ "${file:0:1}" = "@" ] && continue
			[ -n "${dupcheck["$file"]}" ] && continue
			packages["$file"]+=" $module"
			dupcheck["$file"]=1
		done
	done
	missing=( $( filter_installed_packages "${!packages[@]}" ) )
	if [ "${#missing[@]}" -gt 0 ]; then
		pinfo "Making sure required packages are installed..."
		pinfo "Potentially missing: ${missing[*]}"
		install_packages "${missing[@]}"
	fi
	unset_required_vars
}

#
# generic function to read the config file of the current $MODULE
#
read_config () {
	# unset previous variables from other config files
	unset_required_vars

	local MODULE_CONFIG="${MODULE_DIR}/module.conf"

	# sanity checks
	[ ! -e "${MODULE_CONFIG}" ] && perror "Config for '$MODULE' not found."
	validate_config "$MODULE_CONFIG"
	# source the config file
	. "${MODULE_CONFIG}" || perror "Sourcing '${MODULE_CONFIG}' failed."
	# sanity checks - no distribution specific things in the global config
	local CHECK
	for CHECK in REQUIRED_CONTENT_PACKAGES REQUIRED_INSTALLED_PACKAGES; do
		[ -n "${!CHECK}" ] && perror "$CHECK is set in ${MODULE}.conf, but distribution specific config has to go to one of these files:\n$PRINT_SYS_VERSIONS"
	done
	local FILE
	for FILE in $SYS_VERSIONS; do
		if [ -e "${MODULE_CONFIG}.${FILE}" ]; then
			# a specific tool.conf seems to exist, use it to override/extend certain vars
			validate_config "${MODULE_CONFIG}.${FILE}"
			. "${MODULE_CONFIG}.${FILE}" || perror "Sourcing '${MODULE_CONFIG}.${FILE}' failed."
			pinfo "Sourced distribution specific config (${MODULE_CONFIG##*/}.${FILE})"
			return # end function
		fi
	done
	# reaching here means no specific config
	pinfo "Sourced general module config only (${MODULE_CONFIG##*/})"
}

validate_config () {
	local INVALID=$(grep -h -E -o '^\s*\w+=' "$1" | grep -v -E '^\s*REQUIRED_')
	[ -z "$INVALID" ] && return 0
	local i
	for i in $INVALID; do
		pwarning "Invalid variable $i"
	done
	perror "$1 contains invalid variables. All variables must start with REQUIRED_"
}

#
# generic function to read the build script of the current $MODULE
#
read_build () {
	local BUILD_SCRIPT="${MODULE_DIR}/module.build"

	[ ! -e "${BUILD_SCRIPT}" ] && perror "Build script for specified tool not found."

	. "${BUILD_SCRIPT}" || perror "Sourcing '${BUILD_SCRIPT}' failed."
}

export_builds() {
	[ ! -d "${ROOT_DIR}/var/builds" ] && perror "No ${ROOT_DIR}/var/builds, nothing to export."
	if mount | grep -q "${ROOT_DIR}/var/builds"; then
		pwarning "${ROOT_DIR}/var/builds already exported! Ignoring..."
		return 1;
	fi
	pinfo "Mounting ${ROOT_DIR}/var/builds to ${EXPORT_DIR}."
	pinfo "This will make the local var/builds syncable from another machine."
	[ ! -d "${EXPORT_DIR}" ] && mkdir -p "${EXPORT_DIR}"
	mount --bind "${ROOT_DIR}/var/builds" "${EXPORT_DIR}" \
		|| perror "Failed to bind mount ${ROOT_DIR}/var/builds to ${EXPORT_DIR}"
}

#
# main function which copies all files, directories, binaries and external libraries to TARGET_BUILD_DIR
# called after building the module
#
copy_files_with_deps () {

	[ ! -d "$MODULE_BUILD_DIR" ] && pinfo "No build directory found, skipping dependency copying" && return 0
	cde "$MODULE_BUILD_DIR"

	local COPYFILES_LIST="${TARGET_BUILD_DIR}/opt/openslx/.mltk/${MODULE}.copy_files_with_deps"
	local TESTFILE BIN ENTRY OPTIONAL FILENAME LIB LOCATION FILE OLD CLISTCOUNT
	declare -a FILE_CANDIDATES=()
	declare -a FINAL_LIST=()
	rm -f -- "${COPYFILES_LIST}"

	# from REQUIRED_BINARIES - follow symlinks and ldd the file
	[ -n "${REQUIRED_BINARIES}" ] && pinfo "Gathering required binaries from config file..."
	for FILENAME in ${REQUIRED_BINARIES}; do
		if [[ "${FILENAME:0:1}" == "@" ]]; then
			OPTIONAL="yes"
			FILENAME="${FILENAME:1}"
		else
			OPTIONAL="no"
		fi
		mapfile -d '' -t FILE_CANDIDATES < <( find . -name "${FILENAME}" -a \( -type f -o -type l \) -print0 )
		# Check result of find
		if (( ${#FILE_CANDIDATES[@]} > 1 )); then
			# More than one match for binary
			pdebug "Candidates for $FILENAME are: ${FILE_CANDIDATES[*]}"
			for FILE in "${FILE_CANDIDATES[@]}"; do
				TESTFILE="$(readlink -f "$FILE")"
				pdebug "  $FILE leads to $TESTFILE"
				[ -f "$TESTFILE" ] && [ -x "$TESTFILE" ] && FINAL_LIST+=( "$FILE" )
			done
			if (( ${#FINAL_LIST[@]} == 0 )); then
				# Not found in build dir; if install mode, check if it's in the PKG_CONTENT_FILE
				if [ -n "$MLTK_INSTALL" ]; then
					# TODO: Might want to iterate over all the matches and check if at least one of them
					# is +x
					grep -q "/${FILENAME}\$" "$PKG_CONTENT_FILE" && continue
				fi
				perror "\tNo binary found for ${FILENAME}. None of the candidates (${FILE_CANDIDATES[*]}) is executable."
			fi
			if (( ${#FINAL_LIST[@]} > 1 )); then
				pdebug "Found more than one match for required file '$FILENAME': ${FINAL_LIST[*]}"
			else
				pdebug "\tFound ${FILENAME} at ${FILE}"
			fi
		elif [ "$OPTIONAL" == "no" ] && (( ${#FILE_CANDIDATES[@]} == 0 )); then
			# No candidate
			# Same as above: accept existing file in /
			if [ -n "$MLTK_INSTALL" ]; then
				grep -q "/${FILENAME}\$" "$PKG_CONTENT_FILE" && continue
			fi
			perror "Could not find required binary $FILENAME"
		else
			# One candidate
			TESTFILE="$( readlink -f "${FILE_CANDIDATES[0]}" )"
			if [ -f "$TESTFILE" ] && [ -x "$TESTFILE" ]; then
				FINAL_LIST=( "${FILE_CANDIDATES[0]}" )
			elif [ "$OPTIONAL" = "no" ]; then
				perror "No executable regular file found for '$FILENAME' (potential match was ${FILE_CANDIDATES[0]})"
			fi
		fi
		for FILE in "${FINAL_LIST[@]}"; do
			pdebug "* $FILE"
			get_link_chain "${MODULE_BUILD_DIR}/${FILE}" "${MODULE_BUILD_DIR}" >> "${COPYFILES_LIST}"
			get_dynamic_dependencies -l "${MODULE_BUILD_DIR}" "${FILE}" >> "${COPYFILES_LIST}"
		done
	done

	# from REQUIRED_LIBRARIES - like binaries, follow symlinks and ldd, but also expand the lib-name
	# by looking for files named <libname>.so*
	[ -n "$REQUIRED_LIBRARIES" ] && pinfo "Gathering required libraries from config file..."
	for LIB in $REQUIRED_LIBRARIES; do
		mapfile -d '' -t FILE_CANDIDATES < <(find . -name "${LIB}.so*" -print0)
		if (( ${#FILE_CANDIDATES[@]} == 0 )); then
			if [ -n "$MLTK_INSTALL" ]; then
				# TODO: Escape LIB and use a proper regex with -P
				grep -qF "/${LIB}.so" "$PKG_CONTENT_FILE" && continue
			fi
			perror "Cannot find required library $LIB"
		fi
		for LOCATION in "${FILE_CANDIDATES[@]}"; do
			pdebug "* $LOCATION"
			get_link_chain "${MODULE_BUILD_DIR}/${LOCATION}" "${MODULE_BUILD_DIR}"  >> "${COPYFILES_LIST}"
			get_dynamic_dependencies -l "${MODULE_BUILD_DIR}" "${LOCATION}" >> "${COPYFILES_LIST}"
		done
	done

	# from REQUIRED_DIRECTORIES - recursively copy given dirs, look for files that seem to be an elf
	# binary and do the symlink+ldd trick on them
	[ -n "${REQUIRED_DIRECTORIES}" ] && pinfo "Gathering required directories from config file..."
	local ENTRY=""
	for ENTRY in ${REQUIRED_DIRECTORIES}; do
		[[ "$ENTRY" == /* ]] || perror "All entries in REQUIRED_DIRECTORIES have to start with a slash '/', but $ENTRY does not!"
		[ -e "$ENTRY" ] && [ ! -d "$ENTRY" ] && perror "$ENTRY is not a directory in base system"
		pdebug "* $ENTRY"
		if [ -d ".${ENTRY}" ]; then
			ENTRY=".${ENTRY}"
			echo "${ENTRY}" >> "${COPYFILES_LIST}"
			mapfile -d '' -t FILE_CANDIDATES < <(find "${ENTRY}" -type f -a \( -executable -o -name '*.so*' \) -a -not -name '*.a' -print0)
			for BIN in "${FILE_CANDIDATES[@]}"; do
				[ -f "$BIN" ] || continue
				#pdebug "\tSearching libs for ${BIN}..."
				get_link_chain "${MODULE_BUILD_DIR}/${BIN}" "${MODULE_BUILD_DIR}" >> "${COPYFILES_LIST}"
				get_dynamic_dependencies -l "${MODULE_BUILD_DIR}" "${BIN}" >> "${COPYFILES_LIST}"
			done
		elif [ -z "$MLTK_INSTALL" ] || ! [ -d "$ENTRY" ]; then
			perror "Required directory $ENTRY not found"
		fi
	done

	# from REQUIRED_FILES - these are assumed to be simple files, so only follow symlinks
	[ -n "${REQUIRED_FILES}" ] && pinfo "Gathering required files from config file..."
	for ENTRY in ${REQUIRED_FILES}; do
		if [ "${ENTRY:0:1}" = "@" ]; then
			ENTRY="${ENTRY:1}"
			if [ ! -e "${MODULE_BUILD_DIR}/${ENTRY}" ]; then
				pinfo "Skipping missing optional file: ${MODULE_BUILD_DIR}/${ENTRY}"
				continue
			fi
		fi
		if [ -n "$MLTK_INSTALL" ] && ! [ -e "${MODULE_BUILD_DIR}/${ENTRY}" ]; then
			[ -e "/${ENTRY}" ] && continue # OK for install mode... I think
		fi
		get_link_chain "${MODULE_BUILD_DIR}/${ENTRY}" "${MODULE_BUILD_DIR}" >> "${COPYFILES_LIST}"
	done

	#copy to initramfsdir
	pdebug "File list generated at ${COPYFILES_LIST}."
	# empty?
	if [ ! -s "$COPYFILES_LIST" ]; then
		return
	fi
	# unchanged?
	OLD="${COPYFILES_LIST}.old"
	if [ -s "$OLD" ] && cmp -s "$OLD" "$COPYFILES_LIST"; then
		return
	fi
	CLISTCOUNT=$( < "$COPYFILES_LIST"  wc -l )
	pinfo "Copying $CLISTCOUNT files to '${TARGET_BUILD_DIR}'."
	tarcopy "$(sort -u "$COPYFILES_LIST")" "${TARGET_BUILD_DIR}"
	mv -f "$COPYFILES_LIST" "$OLD"
}

#
#
# If the system has no usr split, recreate the structure in the given
# directory, otherwise, do nothing
#
#
prepare_usr_split () {
	local DIR DEST
	local BASE=$1
	[ -z "$BASE" ] && perror "prepare_usr_split called with empty base dir"
	if [ "$USR_SPLIT" == "no" ]; then # no usr split, /bin /lib etc are symlinks
		for DIR in lib lib32 lib64 bin sbin; do
			[ -L "/${DIR}" ] || continue
			DEST=$(readlink "/${DIR}")
			if [ ! -L "/${DEST}" ]; then
				mkdir -p "${BASE}/${DEST}" || perror "Could not create '${BASE}/${DEST}'"
			fi
			[ -L "${BASE}/${DIR}" ] || ln -s "${DEST}" "${BASE}/${DIR}" || perror "Could not symlink '${BASE}/${DIR}' to '${DEST}'"
		done
	fi
}

# Recursively get all modules in the current target
scan_target() {
	local dir item base real
	dir="$1"
	[ -d "$1" ] || perror "scan_target: $dir is not a directory"
	for item in "$dir/"*; do
		[ -L "$item" ] || continue
		base="$( basename "$item" )"
		real="$( readlink -f "$item" )"
		if [ -f "$item/module.conf" ]; then
			if [ -n "${MODULE_PATHS["$base"]}" ] \
					&& [ "${MODULE_PATHS["$base"]}" != "$real" ]; then
				perror "$base exists in multiple sub-targets and links to different modules\n1: $real\n2: ${MODULE_PATHS["$base"]}"
			fi
			MODULE_PATHS["$base"]="$real"
		elif [ "${real#"$TARGETS_BASEDIR"}" != "$real" ]; then
			scan_target "$real"
		else
			perror "Unknown link in target '$dir': '$base' is '$real', neither target nor module"
		fi
	done
}

#
#
# main public function. Requires the TARGET to be given as argument.
# this will simply go over all the modules as found in the core/target/<TARGET>
# and run following functions:
#
#
generate_target() {

	initial_checks

	TARGET="$1"
	shift
	TARGET_DIR="$( readlink -f "${TARGETS_BASEDIR}/${TARGET}" )"
	scan_target "$TARGET_DIR"
	if [[ "$TARGET" == *@* ]]; then
		VARNAME="${TARGET#*@}"
		TARGET="${TARGET%@*}"
		declare -r TARGET_VERSIONS=${!VARNAME}
		if [ -z "$TARGET_VERSIONS" ]; then
			perror "Asked to build versioned target $TARGET with version list in $VARNAME, but $VARNAME is empty."
		fi
		for TARGET_VERSION in ${TARGET_VERSIONS}; do
			TARGET_BUILD_DIR="${ROOT_DIR}/var/builds/${TARGET}@${TARGET_VERSION}"
			generate_target_real "$@"
		done
	else
		declare -r TARGET_VERSIONS=
		declare -r TARGET_VERSION=
		TARGET_BUILD_DIR="${ROOT_DIR}/var/builds/${TARGET}"
		generate_target_real "$@"
	fi
}

generate_target_real() {
	# Keep track of processed modules so we don't
	# build a module multiple times when following dependencies
	local item
	local PROCESSED_MODULES=

	[ -d "$TARGET_DIR" ] || perror "Given target directory does not exist: $TARGET_DIR"

	[[ "$TARGET" == "builds" || "$TARGET" == "modules" ]] && \
		perror "Target directory cannot be named 'builds' or 'modules'."

	pdebug "Generating '$TARGET_BUILD_DIR' for '$TARGET' $TARGET_VERSION"
	mkdir -p "$TARGET_BUILD_DIR" || perror "Failed to create $TARGET_BUILD_DIR"
	prepare_usr_split "${TARGET_BUILD_DIR}"

	# if no arguments assume all.
	if [ -z "$1" ] || [ "$1" = "all" ]; then
		MODULES="${!MODULE_PATHS[*]}"
	else
		# tools = arguments given
		MODULES="$*"
	fi

	pinfo "Activated modules in '${TARGET}':"
	pinfo "\t$(echo ${MODULES})"

	# Make sure all selected modules seem valid
	for item in $MODULES; do
		[ -n "${MODULE_PATHS["$item"]}" ] || perror "Module $item not found in target"
	done

	# Run load hook
	# This will run before installing any packages, which might be required in
	# some cases
	declare -A mod_init_done=()
	for item in $MODULES; do
		process_module_init "$item"
	done

	install_all_packages

	# copy basic libs
	pinfo "Copying libc and ld-linux used by ${SHELL}"
	tarcopy "$(list_basic_libs)" "${TARGET_BUILD_DIR}"

	# now iterate over given tools and copy them
	for item in $MODULES; do
		process_module "$item"
	done

	post_process_target

	pinfo "Target completed. Total size: $(du -bsh "${TARGET_BUILD_DIR}" | awk 'END {print $1}')"
	TOOL_STR=""
}

process_module_init() {
	local item="$1"
	local dep
	mod_init_done["$item"]=1
	unset_required_vars
	. "${MODULE_PATHS["$item"]}/module.conf"
	for dep in $REQUIRED_MODULES; do
		[ -n "${MODULE_PATHS["$dep"]}" ] || perror "Module $dep, dependency of $item, not found in target"
		[ -z "${mod_init_done["$dep"]}" ] && process_module_init "$dep"
	done
	. "${CORE_DIR}/includes/clean_module_funcs.inc" # Clean all hooks, in case the module doesn't define them all
	. "${MODULE_PATHS["$item"]}/module.build"
	# TODO? We only source the general .conf, not the .conf.distro one... needed for module_init?
	module_init
}

process_module() {
	# Parse arguments
	[ "$#" -lt "1" ] && perror "process_module: want >= 1 param."
	if [ "$1" = "--dep-of" ]; then
		shift
		local DEPOF=" $1"
		shift
	else
		local DEPOF=""
	fi
	local MODULE="$1"
	[ -z "$MODULE" ] && perror "No module given when calling process_module"
	[[ "$PROCESSED_MODULES" == *"!${MODULE}!"* ]] && return # Already processed this module
	# Set up dirs ans variables
	PROCESSED_MODULES="${PROCESSED_MODULES}!${MODULE}!"
	local MODULE_DIR="${MODULE_PATHS["${MODULE}"]}"
	local MODULE_WORK_DIR="${ROOT_DIR}/tmp/work/${MODULE}"
	if [[ "$MODULE" == *@ ]]; then
		[ -z "$TARGET_VERSION" ] && perror "Asked to build $MODULE in non-versioned target $TARGET"
		MODULE_WORK_DIR+="-${TARGET_VERSION}"
	fi
	local MODULE_BUILD_DIR="${MODULE_WORK_DIR}/build"
	local MODULE_META_DIR="${MODULE_WORK_DIR}/meta"
	local TOOL_STR="[${MODULE}${TARGET_VERSION}]"
	local SOURCE_FLAG="${MODULE_META_DIR}/fetched_source.flag"
	local BUILD_FLAG="${MODULE_META_DIR}/build_complete.flag"
	local COPY_FLAG="${MODULE_META_DIR}/copy_complete.flag"
	local AUTOCLONE_FILE="${MODULE_META_DIR}/repos"
	local MD5FILE="${TARGET_BUILD_DIR}/opt/openslx/.mltk/${MODULE}.md5"
	local PKG_CONTENT_FILE="${MODULE_META_DIR}/list_package_contents"
	if [ ! -d "${MODULE_DIR}" ]; then
		if [ -z "$DEPOF" ]; then
			perror "Module directory for '$MODULE' not found in ${TARGET_DIR}"
		fi
		perror "Module directory for '$MODULE' not found in ${TARGET_DIR} (is a dependency of${DEPOF})"
		return
	fi
	# Simple check for modified .build / .config, in which case we'll run a module clean first
	if [ -f "$MD5FILE" ] && ! md5sum --check "$MD5FILE"; then
		# Mismatch
		if [ "$MODULE" = "kernel" ]; then
			pwarning " ** Kernel .build/.config changed, but won't autoclean."
			pwarning " ** Check if cleaning should be done and use '-c kernel'"
		else
			pwarning "Module's .build/.config has changed, rebuilding!"
			clean_module "${MODULE}"
		fi
	fi
	# Write new md5 file if not existent yet
	mkdir -p "${TARGET_BUILD_DIR}/opt/openslx/.mltk" "${MODULE_META_DIR}"
	if ! [ -f "$MD5FILE" ]; then
		md5sum "$MODULE_DIR/module".* > "$MD5FILE" || perror "Could not create $MD5FILE"
	fi
	cde "${MODULE_DIR}"
	# Process module
	pdebug "## Reading config of $MODULE"
	read_config
	# Prepare build directory
	mkdir -p "${MODULE_BUILD_DIR}" || perror "Could not create build dir"
	prepare_usr_split "${MODULE_BUILD_DIR}"
	# Check if this module has a dependency that wasn't built yet:
	if [ -n "$REQUIRED_MODULES" ]; then
		pdebug "$MODULE depends on ${REQUIRED_MODULES}...."
		for DEP in $REQUIRED_MODULES; do
			process_module --dep-of "${MODULE}${DEPOF}" "$DEP"
		done
		# Read old config again, as it got overwritten by the deps
		cde "${MODULE_WORK_DIR}"
		read_config
	fi
	[ -n "$DEPOF" ] && local DEPOF_STR="(dependency of${DEPOF})"
	pinfo ">>>>>>>>>>>>>>>>> Processing module [ $MODULE ] $DEPOF_STR"
	# Source .build script of module
	pdebug "## Reading .build of $MODULE"
	. "${CORE_DIR}/includes/clean_module_funcs.inc" # Clean all hooks, in case the module doesn't define them all
	read_build # Read all the hooks from the module
	cde "${MODULE_DIR}"
	# update kernel version variables before running a module, as the last one might have been the kernel...
	get_kernel_version
	# Execute pre_exec before anything else, might change some of the _DIR vars
	pre_exec
	cde "${MODULE_WORK_DIR}"
	# Fetch source code
	if [ ! -e "$SOURCE_FLAG" ]; then
		pinfo "## Fetching source"
		rm -f -- "$AUTOCLONE_FILE"
		fetch_source
		touch "$SOURCE_FLAG" || pwarning "Error setting source-flag"
	fi
	# Build
	if ! [ -e "$BUILD_FLAG" ]; then
		list_packet_files > "$PKG_CONTENT_FILE" || perror "Could not write packet content list to $PKG_CONTENT_FILE"
		# Copy all required packages' contents to build dir, but not in install mode
		if [ -z "$MLTK_INSTALL" ] && [ -s "$PKG_CONTENT_FILE" ]; then
			pinfo "## Copying required packages to build dir"
			tarcopy "$( sort -u "$PKG_CONTENT_FILE" )" "${MODULE_BUILD_DIR}"
		fi
		cde "${MODULE_WORK_DIR}"
		pinfo "## Building"
		build # calls perror if something fails, no need to do that here
		strip_recursive "$MODULE_BUILD_DIR"
		touch "$BUILD_FLAG" || pwarning "Error setting built-flag"
	fi
	# Remove *.la files as they might confuse libtool/linker of other tool packages
	[ -d "${MODULE_BUILD_DIR}" ] && find "${MODULE_BUILD_DIR}" -name '*.la' -exec rm -f {} \;
	# Only copy stuff from system or build dir if md5s didn't change
	if ! [ -e "$COPY_FLAG" ]; then
		pinfo "## Copying files with dependencies"
		cde "${MODULE_DIR}"
		copy_files_with_deps
		if [ -n "$REQUIRED_SYSTEM_FILES" ]; then
			pinfo "## Copying required system files" # REQUIRED_SYSTEM_FILES
			cde "${MODULE_DIR}"
			copy_system_files
		fi
		touch "$COPY_FLAG" || pwarning "Error setting copy-flag"
	fi
	# Always copy static data
	if [ -d "${MODULE_DIR}/data" ]; then
		pinfo "## Copying static module files"
		copy_static_data
	fi
	# TODO: automatic copy of REQUIRED_CONTENT_PACKAGES
	pinfo "## Post copy"
	cde "${MODULE_DIR}"
	post_copy
	[ -s "$AUTOCLONE_FILE" ] && cp "$AUTOCLONE_FILE" "${TARGET_BUILD_DIR}/opt/openslx/.mltk/${MODULE}.git"
	# Sanity checks
	[ -e "$TARGET_BUILD_DIR/var/run" ] &&  ! [ -L "$TARGET_BUILD_DIR/var/run" ] \
		&& perror "Messup datected: $TARGET_BUILD_DIR/var/run exists and is not a symlink!"
	[ -e "$TARGET_BUILD_DIR/var/lock" ] && ! [ -L "$TARGET_BUILD_DIR/var/lock" ] \
		&& perror "Messup datected: $TARGET_BUILD_DIR/var/lock exists and is not a symlink!"
	[ -n "$(ls -A "$TARGET_BUILD_DIR/run" 2>> /dev/null)" ] && perror "Messup detected: $TARGET_BUILD_DIR/run is not empty. You cannot place static files there, use /etc/tmpfiles.d instead!"
}

post_process_target() {
	local TOOL_STR="$TOOL_STR post_process_target:"
	local i

	if [ -z "$MLTK_INSTALL" ]; then
		# traditional minilinux approach
		# figure out all relevant ld-paths
		pinfo "Running ldconfig"
		cp -r -L /etc/ld.so.conf* "${TARGET_BUILD_DIR}/etc/"
		ldconfig -v -r "${TARGET_BUILD_DIR}"

		# run depmod to generate the final modules.* files
		pinfo "Running depmod for '${TARGET_KERNEL_LONG}'"
		depmod -b "${TARGET_BUILD_DIR}" -a "${TARGET_KERNEL_LONG}"
		pinfo "Running glib-compile-schemas"
		for i in /usr/local/share/glib-2.0/schemas/ /usr/share/glib-2.0/schemas/; do
			glib-compile-schemas "${TARGET_BUILD_DIR}/$i"
		done
		return 0
	fi

	# Installer mode
	local RSYNC_DEST RSYNC_EXCLUDE_LIST entry filelist target_file system_file
	# default dest and exclude list for non-addons
	RSYNC_DEST='/'
	RSYNC_EXCLUDE_LIST="$( mktemp "${ROOT_DIR}/tmp/${TARGET}${TARGET_VERSION}-rsync-install-XXXXX" )"
	# always exclude ld.so.cache, we handle that separately
	echo '/etc/ld.so*' > "$RSYNC_EXCLUDE_LIST"
	echo '/autoexec.bat' >> "$RSYNC_EXCLUDE_LIST"
	if [ -e "${TARGET_DIR}/.addon" ]; then
		# This is an addon, don't sync to / but to dedicated directory
		RSYNC_DEST="/opt/openslx/addons/${TARGET_BUILD_DIR##*/}"
		pinfo "Target is an addon, preparing install to '${RSYNC_DEST}'."
		mkdir -p "${RSYNC_DEST}" || perror "Failed to mkdir '${RSYNC_DEST}'."
		RSYNC_OPTS+=( "--delete" "--delete-excluded" )
		cde "$TARGET_BUILD_DIR"
		readarray -t -d '' filelist < <( find . -not -type d -print0 )
		for entry in "${filelist[@]}"; do
			# diff them to be sure they are the same?
			system_file="${entry:1}"
			target_file="${entry:2}"
			if [ -e "$system_file" ]; then
				if [ -L "$system_file" ] && [ -L "$target_file" ]; then
					# Two links with same destination? Skip
					[ "$( readlink "$system_file" )" == "$( readlink "$target_file" )" ] \
						&& echo "$system_file" >> "$RSYNC_EXCLUDE_LIST"
				elif [ -L "$system_file" ] || [ -L "$target_file" ]; then
					: # One is link, the other isn't, do not skip
				elif cmp -s "$system_file" "$target_file"; then
					# Two files with same contents? Skip
					echo "$system_file" >> "$RSYNC_EXCLUDE_LIST"
				fi
			fi
		done
		pinfo "Calling ldconfig on overlay'ed ${TARGET_BUILD_DIR##*/}..."
		chroot_run "${TARGET_BUILD_DIR}" <<< "ldconfig"
		if [ -f "${TARGET_BUILD_DIR}/etc/ld.so.cache" ]; then
			mkdir -p "${TARGET_BUILD_DIR}/opt/openslx/etc"
			mv -f -- "${TARGET_BUILD_DIR}/etc/ld.so.cache" \
				"${TARGET_BUILD_DIR}/opt/openslx/etc/${TARGET_BUILD_DIR##*/}.ld.so.cache"
			pinfo "... generated cache at '${TARGET_BUILD_DIR}/opt/openslx/etc/${TARGET_BUILD_DIR##*/}.ld.so.cache'."
		fi
	fi
	pinfo "Rsyncing local build of '${TARGET_BUILD_DIR}' to '${RSYNC_DEST}'..."
	rsync -aAXv "${RSYNC_OPTS[@]}" \
		--exclude-from="${RSYNC_EXCLUDE_LIST}" \
		"${TARGET_BUILD_DIR}/" "${RSYNC_DEST}" || \
			perror "Failed to rsync, your system is probably trashed ;-("

	[ -z "$MLTK_DEBUG" ] && rm -f -- "$RSYNC_EXCLUDE_LIST"

	# compile Gsettings schema files
	"${RSYNC_DEST}/usr/bin/glib-compile-schemas"  "${RSYNC_DEST}/usr/share/glib-2.0/schemas/"

	# finish by running ldconfig for the running system (when processing non-addons).
	if [ ! -e "${TARGET_DIR}/.addon" ]; then
		pinfo "Running ldconfig"
		ldconfig -v
		pinfo "Running depmod for '${TARGET_KERNEL_LONG}'"
		depmod -a "${TARGET_KERNEL_LONG}"
		pinfo "Running glib-compile-schemas"
		for i in /usr/local/share/glib-2.0/schemas/ /usr/share/glib-2.0/schemas/; do
			glib-compile-schemas "$i"
		done
	fi
}

clean_modules() {
	local module
	TARGET=$1
	shift
	TARGET_DIR="$( readlink -f "${TARGETS_BASEDIR}/${TARGET}" )"
	scan_target "$TARGET_DIR"
	[ -d "$TARGET_DIR" ] || perror "Given target directory does not exist: $TARGET_DIR"

	if [[ "$TARGET" == *@* ]]; then
		VARNAME="${TARGET#*@}"
		TARGET="${TARGET%@*}"
		declare -r TARGET_VERSIONS=${!VARNAME}
		if [ -z "$TARGET_VERSIONS" ]; then
			perror "Asked to build versioned target $TARGET with version list in $VARNAME, but $VARNAME is empty."
		fi
	else
		VARNAME=
		declare -r TARGET_VERSIONS=
	fi

	TARGET_BUILD_BASE="${ROOT_DIR}/var/builds/${TARGET}"

	if [ -z "$1" ] || [ "$1" = "all" ]; then
		if [ -n "$TARGET_VERSIONS" ]; then
			for TARGET_VERSION in ${TARGET_VERSIONS}; do
				rm -rf -- "${TARGET_BUILD_BASE}@${TARGET_VERSION}" || perror "Could not delete target build dirs for $TARGET"
			done
		else
			rm -rf -- "${TARGET_BUILD_BASE}" || perror "Could not delete target build dir for $TARGET"
		fi
		# exclude kernel on "all"
		unset 'MODULE_PATHS[kernel]'
		set -- "${!MODULE_PATHS[@]}"
	fi

	for module in "$@"; do
		[ -n "${MODULE_PATHS["$module"]}" ] || perror "Target does not contain module '$module'"
	done
	cde "$TARGET_DIR"
	for module in "$@"; do
		if [[ "$module" == *@ ]]; then
			[ -z "$TARGET_VERSIONS" ] && perror "Found versioned module $module in non-versioned target $TARGET"
			for TARGET_VERSION in ${TARGET_VERSIONS}; do
				TARGET_BUILD_DIR="${TARGET_BUILD_BASE}@${TARGET_VERSION}"
				clean_module "${module}"
			done
		else
			TARGET_BUILD_DIR="${TARGET_BUILD_BASE}"
			clean_module "${module}"
		fi
	done
}

clean_module() {
	[ -z "$1" ] && perror "No module given on clean_module()"
	[ -z "$TARGET" ] && perror "clean_module called with no active target"
	pinfo "## clean_module $TARGET $1$TARGET_VERSION"

	local MODULE=$1
	local MODULE_DIR="${MODULE_PATHS["${MODULE}"]}"
	local MODULE_WORK_DIR="${ROOT_DIR}/tmp/work/${MODULE}"
	if [[ "$MODULE" == *@ ]]; then
		[ -z "$TARGET_VERSION" ] && perror "Asked to clean $MODULE in non-versioned target $TARGET"
		MODULE_WORK_DIR+="-${TARGET_VERSION}"
	fi

	if ! [ -d "$MODULE_DIR" ]; then
		pwarning "No such module '$MODULE' in current target '$TARGET'"
		pwarning "Check your spelling"
		if [ -d "$MODULE_WORK_DIR" ]; then
			pwarning "Continuing anyways since the according work dir exists..."
		else
			return 1
		fi
	fi

	pinfo "Cleaning '$1'..."
	rm -rf -- "${MODULE_WORK_DIR}" || perror "Could not delete work dir"
	rm -f -- "${TARGET_BUILD_DIR}/opt/openslx/.mltk/${MODULE}."* # no space here!
	return 1
}

# Recursively strip binaries and libraries in the given directory
strip_recursive() {
	local DIR="$1"
	if [ -z "$DIR" ] || ! [ -d "$DIR" ]; then
		perror "strip_recursive(): No such directory: '$DIR'"
	fi
	# Will try to strip shell scripts too but shouldn't do any harm
	# Ignore anything we compile ourselves so we have usable core dumps
	find "$DIR" -type f \! -path "*openslx*bin*" -a \( -executable -o -name "*.so*" \) -exec strip {} \; 2> /dev/null
}

# copies static data files from <MODULE>/data/ to <TARGET_BUILD_DIR>
copy_static_data() {
	[ ! -d "${MODULE_DIR}/data" ] && pinfo "${MODULE} has no static 'data' directory." && return
	cde "${MODULE_DIR}/data/"
	pinfo "tarcopy ${MODULE_DIR}/data/"
	tarcopy "$(find . -type f -o -type l)" "${TARGET_BUILD_DIR}"
}

# Copies files with their absolute paths in $REQUIRED_SYSTEM_FILES to $TARGET_BUILD_DIR
copy_system_files() {
	[ -z "$REQUIRED_SYSTEM_FILES" ] && return
	local file list
	list=
	for file in $REQUIRED_SYSTEM_FILES; do
		list+=" $(get_link_chain "$file")"
	done
	tarcopy "$list" "$TARGET_BUILD_DIR"
}