path: root/remote/setup_target

#!/bin/bash
# -----------------------------------------------------------------------------
# 
# Copyright (c) 2011 - OpenSLX GmbH
#
# This program is free software distributed under the GPL version 2.
# See http://openslx.org/COPYING
#
# If you have any feedback please consult http://openslx.org/feedback and
# send your suggestions, praise, or complaints to feedback@openslx.org
#
# General information about OpenSLX can be found at http://openslx.org/
# ----------------------------------------------------------------------------- 
#
#	This file contains the functions needed to setup a target.
#	The main function is 'generate_target' which will go over all 
#	activated modules (symlinks found under remote/targets/<target>)
#	and run the module-specific 'fetch_source', 'build' and 'post_copy'
#	functions of the modules.
#
#	The functions defined here are module-independent, meaning they
#	can process any module, as long as following requirements are met:
#
#		- Modules need to have a build-script and a config file,
#		  i.e. remote/module/kernel/kernel.build 
#		  and  remote/module/kernel/kernel.conf
#		  as these will be read by 'process_module'.
#		
#		- Modules need to generate a 'build'-directory in their
#		  own sub-directory (i.e. remote/modules/sshd/build/
#		  as 'copy_files_with_deps' will look for the files in
#		  that directory ONLY.
#
#		- Several information have to be set in the config file of
#		  a module:
#		    * REQUIRED_BINARIES - list of binaries
#		    * REQUIRED_FILES - list of regular files
#		    * REQUIRED_DIRECTORIES - list of directories
#		    * REQUIRED_SYSTEM_FILES - files to be copied directly from the system
#		    * REQUIRED_INSTALLED_PACKAGES - list of packages needed at build-time
#		    * REQUIRED_CONTENT_PACKAGES - list of packages where the contents are to be copied
#		    * (optional variables needed by the module-build-script)
#
#		  NOTE: REQUIRED_BINARIES, REQUIRED_FILES, REQUIRED_FILES will be ONLY looked
#		        for in the module's own build directory!
#
#	Optionally, a module can have a static data directory under
#	remote/modules/<module>/data which will be copied as is to the target build directory.
#
#	Moreover modules do not need to specify the dynamic dependencies of given binaries
#	as these will be determined automaticly by the helper function 'get_dynamic_dependencies' 
# -----------------------------------------------------------------------------
#
MODE_DIR="${ROOT_DIR}/remote"
MODULES_DIR="${MODE_DIR}/modules"
EXPORT_DIR="/export/build"
# Keep track of processed modules
PROCESSED_MODULES=""

initial_checks () {
	# Set up distribution and package management
	# Lowercase distributor ID is what we use as the general distribution name
	SYS_DISTRIBUTION=$(lsb_release -is | tolower | sed -r 's/[^a-z0-9]//g;s/project$//g;s/scientificsl$/scientific/g')
	# Then determine packet manager
	case "$SYS_DISTRIBUTION" in
	ubuntu|debian)
		PACKET_MANAGER="apt"
		;;
	opensuse)
		PACKET_MANAGER="zypper"
		;;
	scientific)
		PACKET_MANAGER="yum"
		;;
	*)
		perror "Unknown Distribution: $SYS_DISTRIBUTION - Please specify its packet manager in remote/setup_target"
		;;
	esac
	# Get version - we mangle this quite a bit. first make sure it has no spaces, then split version at period (.), underscore (_) and dash (-)
	local VERSION=$(lsb_release -rs | tolower | sed -r 's/\s//g;s/[\._]/ /g;s/-//g')
	local STRTMP=""
	PRINT_SYS_VERSIONS="*.conf.$SYS_DISTRIBUTION"
	SYS_VERSIONS="$SYS_DISTRIBUTION"
	for PART in $VERSION; do
		[ -z "$PART" ] && continue
		STRTMP+=".$PART"
		SYS_VERSIONS="${SYS_DISTRIBUTION}${STRTMP} $SYS_VERSIONS"
		PRINT_SYS_VERSIONS="*.conf.${SYS_DISTRIBUTION}${STRTMP} $PRINT_SYS_VERSIONS"
	done
	pinfo "Config source order: *.conf first, then the first one of these (if found)"
	pinfo "$PRINT_SYS_VERSIONS"
}

#
# generic function to read the config file of the current $MODULE
#
read_config () {
	# unset previous variables from other config files
	for VARNAME in ${!REQUIRED_*}; do
		unset $VARNAME
	done

	local MODULE_CONFIG="${MODULE_DIR}/${MODULE}.conf"

	# sanity checks
	[ ! -e "${MODULE_CONFIG}" ] && perror "Config for '$MODULE' not found."
	# source the config file
	. "${MODULE_CONFIG}" || perror "Sourcing '${MODULE_CONFIG}' failed."
	# sanity checks - no distribution specific things in the global config
	local CHECK
	for CHECK in REQUIRED_CONTENT_PACKAGES REQUIRED_INSTALLED_PACKAGES; do
		[ -n "${!CHECK}" ] && perror "$CHECK is set in ${MODULE}.conf, but distribution specific config has to go to one of these files:\n$PRINT_SYS_VERSIONS"
	done
	local FILE
	for FILE in $SYS_VERSIONS; do
		if [ -e "${MODULE_CONFIG}.${FILE}" ]; then
			# a specific tool.conf seems to exist, use it to override/extend certain vars
			. "${MODULE_CONFIG}.${FILE}" || perror "Sourcing '${MODULE_CONFIG}.${PACKET_MANAGER}' failed."
			pinfo "Sourced distribution specific ${MODULE_CONFIG}.${FILE}"
			break
		fi
	done
}

#
# generic function to read the build script of the current $MODULE
#
read_build () {
	local BUILD_SCRIPT="${MODULE_DIR}/${MODULE}.build"

	[ ! -e "${BUILD_SCRIPT}" ] && perror "Build script for specified tool not found."

	. "${BUILD_SCRIPT}" || perror "Sourcing '${BUILD_SCRIPT}' failed."
}

export_builds() {
	[ ! -d "${MODE_DIR}/builds" ] && perror "No ${MODE_DIR}/builds, nothing to export."
	pinfo "Mounting ${MODE_DIR}/builds to ${EXPORT_DIR}. This will make the local builds syncable from another machine."
        [ ! -d "${EXPORT_DIR}" ] && mkdir -p "${EXPORT_DIR}"
        mount --bind ${MODE_DIR}/builds ${EXPORT_DIR} || perror "Failed to bind mount ${MODE_DIR}/builds to ${EXPORT_DIR}"
}

#
# main function which copies all files, directories, binaries and external libraries to TARGET_BUILD_DIR
#
copy_files_with_deps () {

	[ ! -d "$MODULE_BUILD_DIR" ] && pinfo "No build directory found, skipping dependency copying" && return 0
	cd "$MODULE_BUILD_DIR"

	COPYFILES_LIST="list_wanted_stage3.2"
	[ -e "${COPYFILES_LIST}" ] && rm "${COPYFILES_LIST}"

		
	[ ! -z "${REQUIRED_BINARIES}" ] && pinfo "Gathering required binaries from config file..."
	for FILENAME in ${REQUIRED_BINARIES}
	do
		local FILE_CANDIDATES=$( find . -name "${FILENAME}" -a \( -type f -o -type l \) )
		# only do if more than one candidate found
		if [ "$(echo $FILE_CANDIDATES | wc -w)" -gt 1 ]; then
			pdebug "Candidates for $FILENAME are: $(echo $FILE_CANDIDATES)"
			local FINAL_LIST=""
			for FILE in $FILE_CANDIDATES; do
				local TESTFILE="$(readlink -f "$FILE")"
				pdebug "  $FILE leads to $TESTFILE"
				[ -f "$TESTFILE" -a -x "$TESTFILE" ] && [ "x$(grep -l -E '^(.ELF|#!)' "$TESTFILE")" != "x" ] && FINAL_LIST="$FINAL_LIST $FILE"
			done
			FINAL_LIST=$(trim "$FINAL_LIST")
			if [ -z "$FINAL_LIST" ]; then
				pwarning "\tNo Binary found for ${FILENAME}. Skipping." 
				continue
			fi
			if [[ "$FINAL_LIST" == *" "* ]]; then
				pwarning "Found more than one match for required file '$FILENAME': $FINAL_LIST"
			else
				pdebug "\tFound ${FILENAME} at ${FILE}"
			fi
		elif [ "$(echo $FILE_CANDIDATES | wc -w)" -eq 0 ]; then
			perror "Could not find required binary $FILENAME"
		else
			# one candidate
			FINAL_LIST=${FILE_CANDIDATES}
		fi
		for FILE in $FINAL_LIST; do
			pdebug "* $FILE"
			strip "$FILE" || pwarning "Could not strip '${FILE}'"
			get_link_chain "${MODULE_BUILD_DIR}/${FILE}" "${MODULE_BUILD_DIR}" >> "${COPYFILES_LIST}"
			get_dynamic_dependencies -l "${MODULE_BUILD_DIR}" "${FILE}" >> "${COPYFILES_LIST}"
		done
	done

	[ ! -z "${REQUIRED_LIBRARIES}" ] && pinfo "Gathering required libraries from config file..."
	for LIB in ${REQUIRED_LIBRARIES}
	do
		for LOCATION in $(find . -name ${LIB}.so\*)
		do
			pdebug "* $LOCATION"
			strip "$LOCATION" || pwarning "Could not strip '${LOCATION}'"
			get_link_chain "${MODULE_BUILD_DIR}/${LOCATION}" "${MODULE_BUILD_DIR}"  >> "${COPYFILES_LIST}"
			get_dynamic_dependencies -l "${MODULE_BUILD_DIR}" "${LOCATION}" >> "${COPYFILES_LIST}"
		done
	done

	[ ! -z "${REQUIRED_DIRECTORIES}" ] && pinfo "Gathering required directories from config file..."
	local CURRENT_PWD=$(pwd) # Prevent calling pwd 50000 times inside the loop below
	for ENTRY in ${REQUIRED_DIRECTORIES}
	do
		pdebug "* .$ENTRY"
		echo "./${ENTRY}" >> "${COPYFILES_LIST}"
		for BIN in $(find ".${ENTRY}" -type f -not -name '*.a' | xargs grep -l '^.ELF')
		do
			#pdebug "\tSearching libs for ${BIN}..."
			get_link_chain "${MODULE_BUILD_DIR}/${BIN}" "${MODULE_BUILD_DIR}" >> "${COPYFILES_LIST}"
			get_dynamic_dependencies -l "${MODULE_BUILD_DIR}" "${BIN}" >> "${COPYFILES_LIST}"
		done
	done
	
	[ ! -z "${REQUIRED_FILES}" ] && pinfo "Gathering required files from config file..."
	for ENTRY in ${REQUIRED_FILES}
	do
		get_link_chain "${MODULE_BUILD_DIR}/${ENTRY}" "${MODULE_BUILD_DIR}" >> "${COPYFILES_LIST}"
		get_dynamic_dependencies -l "${MODULE_BUILD_DIR}" "${MODULE_BUILD_DIR}/.${ENTRY}" >> "${COPYFILES_LIST}"
	done

	#copy to initramfsdir
	pdebug "File list generated at ${MODULE_BUILD_DIR}/${COPYFILES_LIST}."
	if [ -s "$COPYFILES_LIST" ]; then
		local CLISTCOUNT=$(cat "$COPYFILES_LIST" | wc -l)
		pinfo "Copying $CLISTCOUNT files to '${TARGET_BUILD_DIR}'."
		tarcopy "$(cat "$COPYFILES_LIST"|sort -u)" "${TARGET_BUILD_DIR}"
	fi
	
}

#
#
# main public function. Requires the TARGET to be given as argument.
# this will simply go over all the modules as found in the remote/target/<TARGET>
# and run following functions:
# 
#
generate_target() {

	initial_checks
	
	TARGET=$1 && shift
	TARGET_DIR="${MODE_DIR}/targets/${TARGET}"
	TARGET_BUILD_DIR="${MODE_DIR}/builds/${TARGET}"
	[ -e "${ROOT_DIR}/logs/${TARGET}.size" ] && . "${ROOT_DIR}/logs/${TARGET}.size" || echo "declare -A BUILD_SIZE" >> "${ROOT_DIR}/logs/${TARGET}.size" 

	[ -d "$TARGET_DIR" ] || perror "Given target directory does not exist: $TARGET_DIR"

	[[ "$TARGET" == "builds" || "$TARGET" == "modules" ]] && \
		perror "Target directory cannot be named 'builds' or 'modules'."
	
	pinfo "Generating '$TARGET_BUILD_DIR' for '$TARGET'"

	# if no arguments assume all.
	if [ "x$1" = "x" -o "x$1" = "xall" ]; then
		MODULES=$(ls ${TARGET_DIR})
		set -- $MODULES
	else
		# tools = arguments given
		MODULES=$@
	fi

	pinfo "Activated modules in '${TARGET}':"
	pinfo "\t$(echo ${MODULES})"

	# we need a bit of special logic for kernel specifically
	# to check if the built version is corresponding to the
	# current kernel version. If not we had a kernel update
	# and need to ask the user what to do
	if [[ "$MODULES" == *kernel* ]]; then
		# if so, check what kernels have been built
		if [ -e "${MODULES_DIR}/kernel/ksrc/KVERSION" ]; then
			local BUILT_KERNEL_VERSION=$(cat "${MODULES_DIR}/kernel/ksrc/KVERSION")
			if [[ "${BUILT_KERNEL_VERSION}" != "${KERNEL_CURRENT_VERSION}" ]]; then
				pinfo "The built kernel (${BUILT_KERNEL_VERSION}) is older than the running one (${KERNEL_CURRENT_VERSION})!"
				# cleaning the module is enough as it forces to rebuild.
				[[ "${AUTO_UPDATE_KERNEL}" == 1 ]] && pinfo "Auto-Udpate Kernel..." && clean_kernel_module --force
			fi
		elif [ -e "${MODULES_DIR}/kernel/ksrc" ]; then
			# check if ksrc is there, since that would be problematic
			pwarning "${MODULES_DIR}/kernel/ksrc exists, but KVERSION is not found!"
			pwarning "Kernel was probably build with an older version of mltk."
			pwarning "If errors arise, either write your KVERSION manually or clean the kernel."
		fi
	fi

	# copy basic libs
	pinfo "Copying libc and ld-linux used by ${SHELL}"
	tarcopy "$(list_basic_libs)" "${TARGET_BUILD_DIR}"

	[ -d "$TARGET_BUILD_DIR" ] && TARGET_BUILD_SIZE=$(du -bc "${TARGET_BUILD_DIR}" | awk 'END {print $1}') || TARGET_BUILD_SIZE=0
	# now iterate over given tools and copy them
	while (( "$#" )); do
		process_module "$1"
		shift
	done
	
	post_process_target	

	pinfo "Target completed. Total size: $(du -bsh "${TARGET_BUILD_DIR}" | awk 'END {print $1}')"
	TOOL_STR=""
}

process_module() {
	[ "$#" -ne "1" ] && perror "process_module: want 1 param."
	local MODULE="$1"
	[[ "$PROCESSED_MODULES" == *"!${MODULE}!"* ]] && return # Already processed this module
	PROCESSED_MODULES="${PROCESSED_MODULES}!${MODULE}!"
	local MODULE_DIR="${TARGET_DIR}/${MODULE}"
	local MODULE_BUILD_DIR="${MODULE_DIR}/build"
	local TOOL_STR=""
	pinfo ">>>>>>>>>>>>>>>>> Processing module [ $MODULE ]"
	TOOL_STR="[${MODULE}]"
	if [ -d "${MODULE_DIR}" ]; then
		cd "${MODULE_DIR}" || perror "Module dir '${MODULE_DIR}' seems to exist, but cd to it failed."
		pinfo "## Reading config"
		read_config
		# Check if this module has a dependency that wasn't built yet:
		if [ ! -z "$REQUIRED_MODULES" ]; then
			pinfo "$MODULE depends on ${REQUIRED_MODULES}...."
			for DEP in $REQUIRED_MODULES; do
				process_module "$DEP"
			done
			# Read old config again, as it got overwritten by the deps
			cd "${MODULE_DIR}" || perror "Tool dir '${MODULE_DIR}' seems to exist, but cd to it failed (after building deps)."
			read_config
			pinfo "<<<<<<<<<<<<<<<<< Dependency modules processed, back to module [ $MODULE ]"
		fi
		pinfo "## Reading build"
		read_build
		pinfo "## Installing dependencies"
		install_dependencies
		pinfo "## Fetching source"
		[ -e "${MODULE_DIR}/.fetched_source" ] || { fetch_source && touch "${MODULE_DIR}/.fetched_source"; }
		if [ -e "${MODULE_DIR}/.built" -a -d "${MODULE_BUILD_DIR}" ]; then
			pinfo "# built-flag is set, skipping build..."
		else
			pinfo "## Building"
			mkdir -p "${MODULE_BUILD_DIR}" || perror "Could not create build dir"
			build # calls perror if something fails, no need to do that here
			touch "${MODULE_DIR}/.built" || pwarning "Error setting built-flag"
		fi
		# remove *.la files as they might confuse libtool/linker of other tool packages
		[ -d "${MODULE_BUILD_DIR}" ] && find "${MODULE_BUILD_DIR}" -name '*.la' -exec rm -f {} \;
		[ -d "${TARGET_BUILD_DIR}" ] && TARGET_BUILD_SIZE=$(du -bc "${TARGET_BUILD_DIR}" | awk 'END {print $1}') || TARGET_BUILD_SIZE=0
		pinfo "## Copying files with dependencies"
		copy_files_with_deps
		pinfo "## Copying required system files"
		copy_system_files
		pinfo "## Copying static module files"
		copy_static_data
		pinfo "## Post copy"
		post_copy
		# set MODULE_BUILD_SIZE
		calc_size
		pinfo "Module completed. Total size: ${MODULE_BUILD_SIZE}"
	else
		pwarning "Module directory for '$MODULE' not found."
	fi
}

post_process_target() {
	local TOOL_STR="$TOOL_STR post_process_target:"
	# move systemd files from /lib to /etc/lib
	pinfo "move systemd files from /lib/systemd to /etc/systemd"
	if [ -d "${TARGET_BUILD_DIR}/lib/systemd" ]; then
		cd "${TARGET_BUILD_DIR}/lib"
		tarcopy "systemd" "${TARGET_BUILD_DIR}/etc"
		cd - &> /dev/null
		rm -r "${TARGET_BUILD_DIR}/lib/systemd"
	fi

	# replace ck connector with fake one
	# CAVE: On Scientific Lunux (6.4) and perhaps others:
	# If include dbus/dbus-arch-deps.h is missing (file not found) either set a symb. link:
	# /usr/include/dbus-1.0/dbus/dbus-arch-deps.h -> /usr/lib(64)/dbus-1.0/include/dbus/dbus-arch-deps.h
	# or execute "pkg-config dbus-1 --cflags".

	local CK_LIBS=$(find "$TARGET_BUILD_DIR" -name "libck-connector.so*")
	if [ -n "$CK_LIBS" ]; then
		pinfo "Replacing libck-connector with dummy"
		local SRC="$ROOT_DIR/remote/post-patch/ck-connector/fake-ck-connector.c"
		local BIN="$ROOT_DIR/remote/post-patch/ck-connector/libck-connector.so"
		if [ ! -s "$BIN" ]; then
			gcc -O3 -shared -o "$BIN" "-I/usr/include/dbus-1.0" "-I/usr/lib/$ARCH_TRIPLET/dbus-1.0/include" "$SRC" || perror "Compiling fake libck-connector.so failed. Make sure libdbus-1-dev is installed."
		fi
		for OCCURENCE in $CK_LIBS; do
			cp "$BIN" "$OCCURENCE" || perror "Frickeling ck-connector to $OCCURENCE failed."
		done
	fi

	# figure out all relevant ld-paths
	pinfo "Running ldconfig"
	local LDTEMP="$TARGET_BUILD_DIR/ldconf-temp"
	grep -h '^\s*/' /etc/ld.so.conf /etc/ld.so.conf.d/* > "$LDTEMP"
	# TODO: Echo other stuff to $LDTEMP (but figure out if first or last match wins)
	ldconfig -v -r "${TARGET_BUILD_DIR}" -f "/ldconf-temp"
	unlink "$LDTEMP"

}

clean_modules() {

	TARGET=$1
	shift
	TARGET_DIR="${MODE_DIR}/targets/${TARGET}"
	TARGET_BUILD_DIR="${MODE_DIR}/builds/${TARGET}"
	[ -d $TARGET_DIR ] || perror "Given target directory does not exist: $TARGET_DIR"

	if [ "x$1" = "x" -o "x$1" = "xall" ]; then
		if [ -e ${ROOT_DIR}/logs/${TARGET}.size ]; then
			rm "${ROOT_DIR}/logs/${TARGET}.size" || perror "Could not delete logs/${TARGET}.size"
		fi
		if [ -d ${TARGET_BUILD_DIR} ]; then
			pinfo "Cleaning '${TARGET_BUILD_DIR}'" \
				&& rm -rf "${TARGET_BUILD_DIR}"/* \
				|| perror "Error deleting $TARGET_BUILD_DIR"
		fi
		set -- $(ls ${TARGET_DIR} | grep -v kernel)
	fi
	cd "$TARGET_DIR"

	while (( "$#" )); do
		clean_module $(readlink -f $1)
		shift
	done
	cd - &> /dev/null

	# libck-connector cleanup
	rm -f "$TARGET_DIR/post-patch/ck-connector/libck-connector.so"
}

clean_module() {
	# if kernel is to be cleaned, do it separately and return
	[ "x$(basename $1)" == "xkernel" ] && clean_kernel_module && return

	pinfo "Cleaning '$1'..."
	local MODULE_DIR=$1
	if [ -e ${MODULE_DIR}/.built ]; then
		rm "${MODULE_DIR}/.built" || perror "Could not clear built flag"
	fi
	if [ -e ${MODULE_DIR}/.fetched_source ]; then
		rm "${MODULE_DIR}/.fetched_source" || perror "Could not clear fetched_source flag"
	fi
	if [ -d ${MODULE_DIR}/build ]; then
		rm -rf "${MODULE_DIR}/build" || perror "Could not delete build path"
	fi
	if [ -d ${MODULE_DIR}/src ]; then
		rm -rf "${MODULE_DIR}/src" || perror "Could not delete src path"
	fi
	if [ -e ${MODULE_DIR}/list_dpkg_output ]; then
		rm "${MODULE_DIR}/list_dpkg_output" || perror "Could not delete list_dpkg_output"
	fi
	if [ -e ${MODULE_DIR}/list_binaries_and_files ]; then
		rm "${MODULE_DIR}/list_binaries_and_files" || perror "Could not delete list_binaries_and_files"
	fi
}

clean_kernel_module() {
	local CLEAN=0
	if [ "x$1" == "x--force" ]; then
		CLEAN=1
	else
		pinfo "It will take a long time to rebuild, are you sure? [y/N]"
		read USER_INPUT
		[ "x$USER_INPUT" == "xy" ] && CLEAN=1
	fi

	pinfo "Cleaning kernel module (including sources and compiled stuff)."
	if [[ $CLEAN == 1 ]]; then
		cd "${TARGET_DIR}/kernel" || perror "Could not cd to ${TARGET_DIR}/kernel"
		if [ -e build ]; then
			rm -rf build || perror "Could not delete ${TARGET_DIR}/kernel/build"
		fi
		# little workaround to actually list names of entries
		# (if only a dir exists, ls kernel/linux-* would just list
		# the contents of that directory....)
		for FILE in $(ls | grep linux); do
			rm -rf $FILE || perror "Could not delete $TARGET_DIR}/$FILE"
		done
		# finally clean the config generated
		if [ -e openslx.config ]; then
			rm openslx.config || perror "Could not delete ${TARGET_DIR}/kernel/openslx.config"
		fi
		if [ -e .fetched_source ]; then
			rm .fetched_source || perror "Could not delete ${TARGET_DIR}/kernel/.fetched_source"
		fi
		if [ -e .built ]; then
			rm .built || perror "Could not delete ${TARGET_DIR}/kernel/.built"
		fi
		if [ -L ksrc ]; then
			unlink ksrc || perror "Could not unlink ${TARGET_DIR}/kernel/ksrc."
		fi
	else
		pinfo "Not confirmed, not cleaning kernel module."
	fi
	pinfo "Done cleaning kernel."
}