path: root/remote/setup_target

#!/bin/bash
# -----------------------------------------------------------------------------
# 
# Copyright (c) 2011 - OpenSLX GmbH
#
# This program is free software distributed under the GPL version 2.
# See http://openslx.org/COPYING
#
# If you have any feedback please consult http://openslx.org/feedback and
# send your suggestions, praise, or complaints to feedback@openslx.org
#
# General information about OpenSLX can be found at http://openslx.org/
# ----------------------------------------------------------------------------- 
#
#	This file contains the functions needed to setup a target.
#	The main function is 'generate_target' which will go over all 
#	activated modules (symlinks found under remote/targets/<target>)
#	and run the module-specific 'fetch_source', 'build' and 'post_copy'
#	functions of the modules.
#
#	The functions defined here are module-independent, meaning they
#	can process any module, as long as following requirements are met:
#
#		- Modules need to have a build-script and a config file,
#		  i.e. remote/module/kernel/kernel.build 
#		  and  remote/module/kernel/kernel.conf
#		  as these will be read by 'process_module'.
#		
#		- Modules need to generate a 'build'-directory in their
#		  own sub-directory (i.e. remote/modules/sshd/build/
#		  as 'copy_files_with_deps' will look for the files in
#		  that directory ONLY.
#
#		- Several information have to be set in the config file of
#		  a module:
#		    * REQUIRED_BINARIES - list of binaries
#		    * REQUIRED_FILES - list of regular files
#		    * REQUIRED_DIRECTORIES - list of directories
#		    * REQUIRED_SYSTEM_FILES - files to be copied directly from the system
#		    * REQUIRED_INSTALLED_PACKAGES - list of packages needed at build-time
#		    * REQUIRED_CONTENT_PACKAGES - list of packages where the contents are to be copied
#		    * (optional variables needed by the module-build-script)
#
#		  NOTE: REQUIRED_BINARIES, REQUIRED_FILES, REQUIRED_FILES will be ONLY looked
#		        for in the module's own build directory!
#
#	Optionally, a module can have a static data directory under
#	remote/modules/<module>/data which will be copied as is to the target build directory.
#
#	Moreover modules do not need to specify the dynamic dependencies of given binaries
#	as these will be determined automaticly by the helper function 'get_dynamic_dependencies' 
# -----------------------------------------------------------------------------
#
MODE_DIR="${ROOT_DIR}/remote"
MODULES_DIR="${MODE_DIR}/modules"
EXPORT_DIR="/export/build"
# Keep track of processed modules
PROCESSED_MODULES=""

initial_checks () {
	:
}

#
# generic function to read the config file of the current $MODULE
#
read_config () {
	# unset previous variables from other config files
	for VARNAME in ${!REQUIRED_*}; do
		unset $VARNAME
	done

	local MODULE_CONFIG="${MODULE_DIR}/${MODULE}.conf"

	# sanity checks
	[ ! -e "${MODULE_CONFIG}" ] && perror "Config for '$MODULE' not found."
	# source the config file
	. "${MODULE_CONFIG}" || perror "Sourcing '${MODULE_CONFIG}' failed."
	if [ -e "${MODULE_CONFIG}.${PACKET_MANAGER}" ]; then
		# a specific tool.conf seems to exist, use it to override certain vars
		. "${MODULE_CONFIG}.${PACKET_MANAGER}" || perror "Sourcing '${MODULE_CONFIG}.${PACKET_MANAGER}' failed."
	fi
}

#
# generic function to read the build script of the current $MODULE
#
read_build () {
	local BUILD_SCRIPT="${MODULE_DIR}/${MODULE}.build"

	[ ! -e "${BUILD_SCRIPT}" ] && perror "Build script for specified tool not found."

	. "${BUILD_SCRIPT}" || perror "Sourcing '${BUILD_SCRIPT}' failed."
}

export_builds() {
	[ ! -d "${MODE_DIR}/builds" ] && perror "No ${MODE_DIR}/builds, nothing to export."
	pinfo "Mounting ${MODE_DIR}/builds to ${EXPORT_DIR}. This will make the local builds syncable from another machine."
        [ ! -d "${EXPORT_DIR}" ] && mkdir -p "${EXPORT_DIR}"
        mount --bind ${MODE_DIR}/builds ${EXPORT_DIR} || perror "Failed to bind mount ${MODE_DIR}/builds to ${EXPORT_DIR}"
}

#
# main function which copies all files, directories, binaries and external libraries to TARGET_BUILD_DIR
#
copy_files_with_deps () {

	[ ! -d "$MODULE_BUILD_DIR" ] && pinfo "No build directory found, skipping dependency copying" && return 0
	cd "$MODULE_BUILD_DIR"

	COPYFILES_LIST="list_wanted_stage3.2"
	[ -e "${COPYFILES_LIST}" ] && rm "${COPYFILES_LIST}"

		
	[ ! -z "${REQUIRED_BINARIES}" ] && pinfo "Gathering required binaries from config file..."
	for FILENAME in ${REQUIRED_BINARIES}
	do
		local FILE_CANDIDATES=$( find . -name "${FILENAME}" -a \( -type f -o -type l \) )
		# only do if more than one candidate found
		if [ "$(echo $FILE_CANDIDATES | wc -w)" -gt 1 ]; then
			pdebug "Candidates for $FILENAME are: $(echo $FILE_CANDIDATES)"
			local FINAL_LIST=""
			for FILE in $FILE_CANDIDATES; do
				local TESTFILE="$(readlink -f "$FILE")"
				pdebug "  $FILE leads to $TESTFILE"
				[ -f "$TESTFILE" -a -x "$TESTFILE" ] && [ "x$(grep -l -E '^(.ELF|#!)' "$TESTFILE")" != "x" ] && FINAL_LIST="$FINAL_LIST $FILE"
			done
			FINAL_LIST=$(trim "$FINAL_LIST")
			if [ -z "$FINAL_LIST" ]; then
				pwarning "\tNo Binary found for ${FILENAME}. Skipping." 
				continue
			fi
			if [[ "$FINAL_LIST" == *" "* ]]; then
				pwarning "Found more than one match for required file '$FILENAME': $FINAL_LIST"
			else
				pdebug "\tFound ${FILENAME} at ${FILE}"
			fi
		else
			# one candidate
			FINAL_LIST=${FILE_CANDIDATES}
		fi
		for FILE in $FINAL_LIST; do
			pdebug "* $FILE"
			strip "$FILE" || pwarning "Could not strip '${FILE}'"
			get_link_chain "${MODULE_BUILD_DIR}/${FILE}" "${MODULE_BUILD_DIR}" >> "${COPYFILES_LIST}"
			get_dynamic_dependencies -l "${MODULE_BUILD_DIR}" "${FILE}" >> "${COPYFILES_LIST}"
		done
	done

	[ ! -z "${REQUIRED_LIBRARIES}" ] && pinfo "Gathering required libraries from config file..."
	for LIB in ${REQUIRED_LIBRARIES}
	do
		for LOCATION in $(find . -name ${LIB}.so\*)
		do
			pdebug "* $LOCATION"
			strip "$LOCATION" || pwarning "Could not strip '${LOCATION}'"
			get_link_chain "${MODULE_BUILD_DIR}/${LOCATION}" "${MODULE_BUILD_DIR}"  >> "${COPYFILES_LIST}"
			get_dynamic_dependencies -l "${MODULE_BUILD_DIR}" "${LOCATION}" >> "${COPYFILES_LIST}"
		done
	done

	[ ! -z "${REQUIRED_DIRECTORIES}" ] && pinfo "Gathering required directories from config file..."
	local CURRENT_PWD=$(pwd) # Prevent calling pwd 50000 times inside the loop below
	for ENTRY in ${REQUIRED_DIRECTORIES}
	do
		pdebug "* .$ENTRY"
		echo "./${ENTRY}" >> "${COPYFILES_LIST}"
		for BIN in $(find ".${ENTRY}" -type f -not -name '*.a' | xargs grep -l '^.ELF')
		do
			#pdebug "\tSearching libs for ${BIN}..."
			get_link_chain "${MODULE_BUILD_DIR}/${BIN}" "${MODULE_BUILD_DIR}" >> "${COPYFILES_LIST}"
			get_dynamic_dependencies -l "${MODULE_BUILD_DIR}" "${BIN}" >> "${COPYFILES_LIST}"
		done
	done
	
	[ ! -z "${REQUIRED_FILES}" ] && pinfo "Gathering required files from config file..."
	for ENTRY in ${REQUIRED_FILES}
	do
		get_link_chain "${MODULE_BUILD_DIR}/${ENTRY}" "${MODULE_BUILD_DIR}" >> "${COPYFILES_LIST}"
		get_dynamic_dependencies -l "${MODULE_BUILD_DIR}" "${MODULE_BUILD_DIR}/.${ENTRY}" >> "${COPYFILES_LIST}"
	done

	#copy to initramfsdir
	pdebug "File list generated at ${MODULE_BUILD_DIR}/${COPYFILES_LIST}."
	if [ -s "$COPYFILES_LIST" ]; then
		local CLISTCOUNT=$(cat "$COPYFILES_LIST" | wc -l)
		pinfo "Copying $CLISTCOUNT files to '${TARGET_BUILD_DIR}'."
		tarcopy "$(cat "$COPYFILES_LIST"|sort -u)" "${TARGET_BUILD_DIR}"
	fi
	
}

#
#
# main public function. Requires the TARGET to be given as argument.
# this will simply go over all the modules as found in the remote/target/<TARGET>
# and run following functions:
# 
#
generate_target() {

	initial_checks
	
	TARGET=$1 && shift
	TARGET_DIR="${MODE_DIR}/targets/${TARGET}"
	TARGET_BUILD_DIR="${MODE_DIR}/builds/${TARGET}"
	[ -e "${ROOT_DIR}/logs/${TARGET}.size" ] && . "${ROOT_DIR}/logs/${TARGET}.size" || echo "declare -A BUILD_SIZE" >> "${ROOT_DIR}/logs/${TARGET}.size" 

	[ -d "$TARGET_DIR" ] || perror "Given target directory does not exist: $TARGET_DIR"

	[[ "$TARGET" == "builds" || "$TARGET" == "modules" ]] && \
		perror "Target directory cannot be named 'builds' or 'modules'."
	
	pinfo "Generating '$TARGET_BUILD_DIR' for '$TARGET'"

	# if no arguments assume all.
	if [ "x$1" = "x" -o "x$1" = "xall" ]; then
		MODULES=$(ls ${TARGET_DIR})
		set -- $MODULES
	else
		# tools = arguments given
		MODULES=$@
	fi

	pinfo "Activated modules in '${TARGET}':"
	pinfo "\t$(echo ${MODULES})"

	# copy basic libs
	pinfo "Copying libc and ld-linux used by ${SHELL}"
	tarcopy "$(list_basic_libs)" "${TARGET_BUILD_DIR}"

	[ -d "$TARGET_BUILD_DIR" ] && TARGET_BUILD_SIZE=$(du -bc "${TARGET_BUILD_DIR}" | awk 'END {print $1}') || TARGET_BUILD_SIZE=0
	# now iterate over given tools and copy them
	while (( "$#" )); do
		process_module "$1"
		shift
	done
	
	post_process_target	

	pinfo "Target completed. Total size: $(du -bsh "${TARGET_BUILD_DIR}" | awk 'END {print $1}')"
	TOOL_STR=""
}

process_module() {
	[ "$#" -ne "1" ] && perror "process_module: want 1 param."
	local MODULE="$1"
	[[ "$PROCESSED_MODULES" == *"!${MODULE}!"* ]] && return # Already processed this module
	PROCESSED_MODULES="${PROCESSED_MODULES}!${MODULE}!"
	local MODULE_DIR="${TARGET_DIR}/${MODULE}"
	local MODULE_BUILD_DIR="${MODULE_DIR}/build"
	local TOOL_STR=""
	pinfo ">>>>>>>>>>>>>>>>> Processing module [ $MODULE ]"
	TOOL_STR="[${MODULE}]"
	if [ -d "${MODULE_DIR}" ]; then

		#[ "x$DEBUG" != "x1" ] \
		#	&& echo "Logging to ${TOOL_DIR}/${TOOL}/stage32.log" \
		#	&& exec 6>&1 > ${TOOL_DIR}/${TOOL}/stage32.log
		# TODO: Make above work with the new logging system (add function to logging.inc to switch logfile)
		cd "${MODULE_DIR}" || perror "Module dir '${MODULE_DIR}' seems to exist, but cd to it failed."
		pinfo "## Reading config"
		read_config
		# Check if this module has a dependency that wasn't built yet:
		if [ ! -z "$REQUIRED_MODULES" ]; then
			pinfo "$MODULE depends on ${REQUIRED_MODULES}...."
			for DEP in $REQUIRED_MODULES; do
				process_module "$DEP"
			done
			# Read old config again, as it got overwritten by the deps
			cd "${MODULE_DIR}" || perror "Tool dir '${MODULE_DIR}' seems to exist, but cd to it failed (after building deps)."
			read_config
			pinfo "<<<<<<<<<<<<<<<<< Dependency modules processed, back to module [ $MODULE ]"
		fi
		pinfo "## Reading build"
		read_build
		pinfo "## Installing dependencies"
		install_dependencies
		pinfo "## Fetching source"
		[ -e "${MODULE_DIR}/.fetched_source" ] || { fetch_source && touch "${MODULE_DIR}/.fetched_source"; }
		if [ -e "${MODULE_DIR}/.built" -a -d "${MODULE_BUILD_DIR}" ]; then
			pinfo "# built-flag is set, skipping build..."
		else
			pinfo "## Building"
			mkdir -p "${MODULE_BUILD_DIR}" || perror "Could not create build dir"
			build # calls perror if something fails, no need to do that here
			touch "${MODULE_DIR}/.built" || pwarning "Error setting built-flag"
		fi
		# remove *.la files as they might confuse libtool/linker of other tool packages
		[ -d "${MODULE_BUILD_DIR}" ] && find "${MODULE_BUILD_DIR}" -name '*.la' -exec rm -f {} \;
		[ -d "${TARGET_BUILD_DIR}" ] && TARGET_BUILD_SIZE=$(du -bc "${TARGET_BUILD_DIR}" | awk 'END {print $1}') || TARGET_BUILD_SIZE=0
		pinfo "## Copying files with dependencies"
		copy_files_with_deps
		pinfo "## Copying required system files"
		copy_system_files
		pinfo "## Copying static module files"
		copy_static_data
		pinfo "## Post copy"
		post_copy
		# reset pipes
		#[ "x$DEBUG" != "x1" ] && exec 1>&6 6>&- 
		# TODO
		calc_size #sets MODULE_BUILD_SIZE
		pinfo "Module completed. Total size: ${MODULE_BUILD_SIZE}"
	else
		pwarning "Module directory for '$MODULE' not found."
	fi
}

post_process_target() {

	local TOOL_STR="$TOOL_STR post_process_target:"

	#move systemd files from /lib to /etc/lib
	pinfo "move systemd files from /lib/systemd to /etc/systemd"

	if [ -d "${TARGET_BUILD_DIR}/lib/systemd" ]; then
		cd "${TARGET_BUILD_DIR}/lib"
		tarcopy "systemd" "${TARGET_BUILD_DIR}/etc"
		cd - &> /dev/null
		rm -r "${TARGET_BUILD_DIR}/lib/systemd"
	fi

	# figure out all relevant ld-paths
	local LDTEMP="$TARGET_BUILD_DIR/ldconf-temp"
	grep -h '^\s*/' /etc/ld.so.conf /etc/ld.so.conf.d/* > "$LDTEMP"
	# TODO: Echo other stuff to $LDTEMP (but figure out if first or last match wins)
	ldconfig -v -r "${TARGET_BUILD_DIR}" -f "/ldconf-temp"
	unlink "$LDTEMP"

}

clean_modules() {

	TARGET=$1
	shift
	TARGET_DIR="${MODE_DIR}/targets/${TARGET}"
	TARGET_BUILD_DIR="${MODE_DIR}/builds/${TARGET}"
	[ -d $TARGET_DIR ] || perror "Given target directory does not exist: $TARGET_DIR"

	if [ "x$1" = "x" -o "x$1" = "xall" ]; then
		if [ -e ${ROOT_DIR}/logs/${TARGET}.size ]; then
			rm "${ROOT_DIR}/logs/${TARGET}.size" || perror "Could not delete logs/${TARGET}.size"
		fi
		if [ -d ${TARGET_BUILD_DIR} ]; then
			pinfo "Cleaning '${TARGET_BUILD_DIR}'" \
				&& rm -rf "${TARGET_BUILD_DIR}"/* \
				|| perror "Error deleting $TARGET_BUILD_DIR"
		fi
		set -- $(ls ${TARGET_DIR} | grep -v kernel)
	fi
	cd ${TARGET_DIR}

	while (( "$#" )); do
		clean_module $(readlink -f $1)
		shift
	done
	cd - &> /dev/null
}

clean_module() {
	# if kernel is to be cleaned, do it separately and return
	[ "x$(basename $1)" == "xkernel" ] && clean_kernel_module && return

	pinfo "Cleaning '$1'..."
	local MODULE_DIR=$1
	if [ -e ${MODULE_DIR}/.built ]; then
		rm "${MODULE_DIR}/.built" || perror "Could not clear built flag"
	fi
	if [ -e ${MODULE_DIR}/.fetched_source ]; then
		rm "${MODULE_DIR}/.fetched_source" || perror "Could not clear fetched_source flag"
	fi
	if [ -d ${MODULE_DIR}/build ]; then
		rm -rf "${MODULE_DIR}/build" || perror "Could not delete build path"
	fi
	if [ -d ${MODULE_DIR}/src ]; then
		rm -rf "${MODULE_DIR}/src" || perror "Could not delete src path"
	fi
	if [ -e ${MODULE_DIR}/list_dpkg_output ]; then
		rm "${MODULE_DIR}/list_dpkg_output" || perror "Could not delete list_dpkg_output"
	fi
	if [ -e ${MODULE_DIR}/list_binaries_and_files ]; then
		rm "${MODULE_DIR}/list_binaries_and_files" || perror "Could not delete list_binaries_and_files"
	fi
}

clean_kernel_module() {
	pinfo "Cleaning kernel module (including sources and compiled stuff)."
	pinfo "It will take a long time to rebuild, are you sure? [y/N]"
	read USER_INPUT
	if [ "x$USER_INPUT" == "xy" ]; then
		pinfo "Confirmed, going on..."
		# clean kernel module
		cd "${TARGET_DIR}/kernel" || perror "Could not cd to ${TARGET_DIR}/kernel"
		if [ -e build ]; then
			rm -rf build || perror "Could not delete ${TARGET_DIR}/kernel/build"
		fi
		# little workaround to actually list names of entries
		# (if only a dir exists, ls kernel/linux-* would just list
		# the contents of that directory....)
		for FILE in $(ls | grep linux); do
			rm -rf $FILE || perror "Could not delete $TARGET_DIR}/$FILE"
		done
		# finally clean the config generated
		if [ -e openslx.config ]; then
			rm openslx.config || perror "Could not delete ${TARGET_DIR}/kernel/openslx.config"
		fi
		if [ -e .fetched_source ]; then
			rm .fetched_source || perror "Could not delete ${TARGET_DIR}/kernel/.fetched_source"
		fi
		if [ -e .built ]; then
			rm .built || perror "Could not delete ${TARGET_DIR}/kernel/.built"
		fi
		if [ -e ksrc ]; then
			unlink ksrc || perror "Could not unlink ${TARGET_DIR}/kernel/ksrc."
		fi
	else
		pinfo "Not confirmed, not cleaning kernel module."
	fi
	pinfo "Done cleaning kernel."
}