<setup_target> Add check for changed .build/.conf file of modules, autoclean if changed

This should help messups through changed .config or .build scripts where some parts
of the build process have been changed, but fetch_source() or build() wouldn't be called
since the according flag files are present. In case the files didn't change and the
module has already been built, copy_files_with_deps is now skipped too.

Also changed the location of theese flag files. They're in $MODULE_BUILD_DIR now, so
the module's root dir won't be cluttered. Should be applied to 'list_dpkg_output'
and 'list_binaries_and_files' too at some point.

1 files changed, 110 insertions, 65 deletions

diff --git a/remote/setup_target b/remote/setup_target
index be35e0fe..8f9b9708 100755
--- a/remote/setup_target
+++ b/remote/setup_target
@@ -134,44 +134,50 @@ copy_files_with_deps () {
 	[ ! -d "$MODULE_BUILD_DIR" ] && pinfo "No build directory found, skipping dependency copying" && return 0
 	cd "$MODULE_BUILD_DIR"
 
-	COPYFILES_LIST="list_wanted_stage3.2"
+	local COPYFILES_LIST="$MODULE_BUILD_DIR/copy_files_with_deps.filelist"
 	[ -e "${COPYFILES_LIST}" ] && rm "${COPYFILES_LIST}"
 
+	# from REQUIRED_BINARIES - follow symlinks and ldd the file
 	[ ! -z "${REQUIRED_BINARIES}" ] && pinfo "Gathering required binaries from config file..."
 	local OPTIONAL
-	for FILENAME in ${REQUIRED_BINARIES}
-	do
+	for FILENAME in ${REQUIRED_BINARIES}; do
 		if [[ "$FILENAME" == @* ]]; then
 			OPTIONAL="yes"
 			FILENAME=$(echo "$FILENAME" | cut -c 2- )
 		else
 			OPTIONAL="no"
 		fi
-		local FILE_CANDIDATES=$( find . -name "${FILENAME}" -a \( -type f -o -type l \) )
-		# only do if more than one candidate found
-		if [ "$(echo $FILE_CANDIDATES | wc -w)" -gt 1 ]; then
+		local FILE_CANDIDATES=$( find . -name "${FILENAME}" -a -executable -a \( -type f -o -type l \) )
+		local FINAL_LIST=""
+		# Check result of find
+		if [ "$(echo "$FILE_CANDIDATES" | wc -l)" -gt 1 ]; then
+			# More than one match for binary
 			pdebug "Candidates for $FILENAME are: $(echo $FILE_CANDIDATES)"
-			local FINAL_LIST=""
 			for FILE in $FILE_CANDIDATES; do
 				local TESTFILE="$(readlink -f "$FILE")"
 				pdebug "  $FILE leads to $TESTFILE"
-				[ -f "$TESTFILE" -a -x "$TESTFILE" ] && [ "x$(grep -l -E '^(.ELF|#!)' "$TESTFILE")" != "x" ] && FINAL_LIST="$FINAL_LIST $FILE"
+				[ -f "$TESTFILE" -a -x "$TESTFILE" ] && FINAL_LIST="$FINAL_LIST $FILE"
 			done
 			FINAL_LIST=$(trim "$FINAL_LIST")
 			if [ -z "$FINAL_LIST" ]; then
-				pwarning "\tNo Binary found for ${FILENAME}. Skipping."
-				continue
+				perror "\tNo binary found for ${FILENAME}. None of the candidates ($FILE_CANDIDATES) is executable."
 			fi
 			if [[ "$FINAL_LIST" == *" "* ]]; then
 				pdebug "Found more than one match for required file '$FILENAME': $FINAL_LIST"
 			else
 				pdebug "\tFound ${FILENAME} at ${FILE}"
 			fi
-		elif [ "$OPTIONAL" == "no" -a "$(echo $FILE_CANDIDATES | wc -w)" -eq 0 ]; then
+		elif [ "$OPTIONAL" == "no" -a -z "$FILE_CANDIDATES" ]; then
+			# No candidate
 			perror "Could not find required binary $FILENAME"
 		else
-			# one candidate
-			FINAL_LIST=${FILE_CANDIDATES}
+			# One candidate
+			local TESTFILE=$(readlink -f "$FILE_CANDIDATES")
+			if [ -f "$TESTFILE" -a -x "$TESTFILE" ]; then
+				FINAL_LIST=${FILE_CANDIDATES}
+			elif [ "$OPTIONAL" = "no" ]; then
+				perror "No executable regular file found for '$FILENAME' (potential match was $(echo $FILE_CANDIDATES))"
+			fi
 		fi
 		for FILE in $FINAL_LIST; do
 			pdebug "* $FILE"
@@ -181,6 +187,8 @@ copy_files_with_deps () {
 		done
 	done
 
+	# from REQUIRED_LIBRARIES - like binaries, follow symlinks and ldd, but also expand the lib-name
+	# by looking for files named <libname>.so*
 	[ ! -z "$REQUIRED_LIBRARIES" ] && pinfo "Gathering required libraries from config file..."
 	for LIB in $REQUIRED_LIBRARIES; do
 		FILE_CANDIDATES=$(find . -name "${LIB}.so*")
@@ -193,33 +201,44 @@ copy_files_with_deps () {
 		done
 	done
 
+	# from REQUIRED_DIRECTORIES - recursively copy given dirs, look for files that seem to be an elf
+	# binary and do the symlink+ldd trick on them
 	[ ! -z "${REQUIRED_DIRECTORIES}" ] && pinfo "Gathering required directories from config file..."
-	local CURRENT_PWD=$(pwd) # Prevent calling pwd 50000 times inside the loop below
+	local ENTRY=""
 	for ENTRY in ${REQUIRED_DIRECTORIES}; do
-		pdebug "* .$ENTRY"
-		echo "./${ENTRY}" >> "${COPYFILES_LIST}"
-		for BIN in $(find ".${ENTRY}" -type f -not -name '*.a' | xargs grep -l '^.ELF')
-		do
+		[[ "$ENTRY" == /* ]] || perror "All entries in REQUIRED_DIRECTORIES have to start with a slash '/', but $ENTRY does not!"
+		pdebug "* $ENTRY"
+		ENTRY=".${ENTRY}"
+		echo "${ENTRY}" >> "${COPYFILES_LIST}"
+		for BIN in $(find "${ENTRY}" -type f -a \( -executable -o -name '*.so*' \) -a -not -name '*.a'); do
 			#pdebug "\tSearching libs for ${BIN}..."
+			strip "$BIN" || pdebug "Could not strip '${LOCATION}'"
 			get_link_chain "${MODULE_BUILD_DIR}/${BIN}" "${MODULE_BUILD_DIR}" >> "${COPYFILES_LIST}"
 			get_dynamic_dependencies -l "${MODULE_BUILD_DIR}" "${BIN}" >> "${COPYFILES_LIST}"
 		done
 	done
 
+	# from REQUIRED_FILES - these are assumed to be simple files, so only follow symlinks
 	[ ! -z "${REQUIRED_FILES}" ] && pinfo "Gathering required files from config file..."
 	for ENTRY in ${REQUIRED_FILES}; do
 		get_link_chain "${MODULE_BUILD_DIR}/${ENTRY}" "${MODULE_BUILD_DIR}" >> "${COPYFILES_LIST}"
-		get_dynamic_dependencies -l "${MODULE_BUILD_DIR}" "${MODULE_BUILD_DIR}/.${ENTRY}" >> "${COPYFILES_LIST}"
 	done
 
 	#copy to initramfsdir
 	pdebug "File list generated at ${MODULE_BUILD_DIR}/${COPYFILES_LIST}."
-	if [ -s "$COPYFILES_LIST" ]; then
-		local CLISTCOUNT=$(cat "$COPYFILES_LIST" | wc -l)
-		pinfo "Copying $CLISTCOUNT files to '${TARGET_BUILD_DIR}'."
-		tarcopy "$(cat "$COPYFILES_LIST"|sort -u)" "${TARGET_BUILD_DIR}"
+	# empty?
+	if [ ! -s "$COPYFILES_LIST" ]; then
+		return
 	fi
-
+	# unchanged?
+	local OLD="${COPYFILES_LIST}.old"
+	if [ -s "$OLD" ] && diff "$OLD" "$COPYFILES_LIST"; then
+		return
+	fi
+	local CLISTCOUNT=$(cat "$COPYFILES_LIST" | wc -l)
+	pinfo "Copying $CLISTCOUNT files to '${TARGET_BUILD_DIR}'."
+	tarcopy "$(sort -u "$COPYFILES_LIST")" "${TARGET_BUILD_DIR}"
+	mv -f "$COPYFILES_LIST" "$OLD"
 }
 
 #
@@ -300,6 +319,7 @@ generate_target() {
 }
 
 process_module() {
+	# Parse arguments
 	[ "$#" -lt "1" ] && perror "process_module: want >= 1 param."
 	if [ "x$1" = "x--dep-of" ]; then
 		shift
@@ -311,11 +331,14 @@ process_module() {
 	local MODULE="$1"
 	[ -z "$MODULE" ] && perror "No module given when calling process_module"
 	[[ "$PROCESSED_MODULES" == *"!${MODULE}!"* ]] && return # Already processed this module
+	# Set up dirs ans variables
 	PROCESSED_MODULES="${PROCESSED_MODULES}!${MODULE}!"
 	local MODULE_DIR="${TARGET_DIR}/${MODULE}"
 	local MODULE_BUILD_DIR="${MODULE_DIR}/build"
-	local TOOL_STR=""
-	TOOL_STR="[${MODULE}]"
+	local TOOL_STR="[${MODULE}]"
+	local SOURCE_FLAG="$MODULE_BUILD_DIR/fetched_source.flag"
+	local BUILD_FLAG="$MODULE_BUILD_DIR/build_complete.flag"
+	local MD5FILE="$MODULE_BUILD_DIR/module_checksum.md5"
 	if [ ! -d "${MODULE_DIR}" ]; then
 		if [ -z "$DEPOF" ]; then
 			perror "Module directory for '$MODULE' not found in ${TAREGET_DIR}"
@@ -324,6 +347,25 @@ process_module() {
 		return
 	fi
 	cd "${MODULE_DIR}" || perror "Module dir '${MODULE_DIR}' seems to exist, but cd to it failed."
+	# Simple check for modified .build / .config, in which case we'll run a module clean first
+	if [ -f "$MD5FILE" ]; then
+		if ! md5sum --check "$MD5FILE"; then
+			# Mismatch
+			if [ "x$MODULE" = "xkernel" ]; then
+				pwarning " ** Kernel .build/.config changed, but won't autoclean."
+				pwarning " ** Check if cleaning should be done and use '-c kernel'"
+			else
+				pwarning "Module's .build/.config has changed, rebuilding!"
+				clean_module "$MODULE_DIR"
+			fi
+		elif [ ! -e "$SOURCE_FLAG" ] || [ ! -e "$BUILD_FLAG" ]; then
+			# One of the flags is missing, in this case delete the .md5 file
+			# which would prevent dependency copying etc. below
+			rm -f -- "$MD5FILE"
+		fi
+	fi
+	# Process module
+	mkdir -p "${MODULE_BUILD_DIR}" || perror "Could not create build dir"
 	pdebug "## Reading config of $MODULE"
 	read_config
 	# Check if this module has a dependency that wasn't built yet:
@@ -338,38 +380,40 @@ process_module() {
 	fi
 	[ -n "$DEPOF" ] && local DEPOF_STR="(dependency of${DEPOF})"
 	pinfo ">>>>>>>>>>>>>>>>> Processing module [ $MODULE ] $DEPOF_STR"
-	pdebug "## Reading build of $MODULE"
+	# Source .build script of module
+	pdebug "## Reading .build of $MODULE"
 	read_build
+	# Install module's dependencies
 	pdebug "## Installing dependencies"
 	cd "${MODULE_DIR}" || perror "cd to '${MODULE_DIR}' failed."
 	install_dependencies
-	if [ ! -e "${MODULE_DIR}/.fetched_source" ]; then
+	# Fetch source code
+	if [ ! -e "$SOURCE_FLAG" ]; then
 		pinfo "## Fetching source"
-		fetch_source && touch "${MODULE_DIR}/.fetched_source"
+		fetch_source
+		touch "$SOURCE_FLAG" || pwarning "Error setting source-flag"
 	fi
-	if [ ! -e "${MODULE_DIR}/.built" -a -d "${MODULE_BUILD_DIR}" ]; then
+	# Build
+	if [ ! -e "$BUILD_FLAG" ]; then
 		pinfo "## Building"
-		mkdir -p "${MODULE_BUILD_DIR}" || perror "Could not create build dir"
 		cd "${MODULE_DIR}" || perror "cd to '${MODULE_DIR}' failed."
 		build # calls perror if something fails, no need to do that here
-		touch "${MODULE_DIR}/.built" || pwarning "Error setting built-flag"
+		touch "$BUILD_FLAG" || pwarning "Error setting built-flag"
 	fi
-	# remove *.la files as they might confuse libtool/linker of other tool packages
+	# Remove *.la files as they might confuse libtool/linker of other tool packages
 	[ -d "${MODULE_BUILD_DIR}" ] && find "${MODULE_BUILD_DIR}" -name '*.la' -exec rm -f {} \;
-	# update size of target build dir
-	if [ -d "${TARGET_BUILD_DIR}" ]; then
-		TARGET_BUILD_SIZE=$(du -bc "${TARGET_BUILD_DIR}" | awk 'END {print $1}')
-	else
-		TARGET_BUILD_SIZE=0
-	fi
-	pinfo "## Copying files with dependencies"
-	cd "${MODULE_DIR}" || perror "cd to '${MODULE_DIR}' failed."
-	copy_files_with_deps
-	if [ -n "$REQUIRED_SYSTEM_FILES" ]; then
-		pinfo "## Copying required system files" # REQUIRED_SYSTEM_FILES
+	# Only copy stuff from system or build dir if md5s didn't change
+	if [ ! -f "$MD5FILE" ]; then
+		pinfo "## Copying files with dependencies"
 		cd "${MODULE_DIR}" || perror "cd to '${MODULE_DIR}' failed."
-		copy_system_files
+		copy_files_with_deps
+		if [ -n "$REQUIRED_SYSTEM_FILES" ]; then
+			pinfo "## Copying required system files" # REQUIRED_SYSTEM_FILES
+			cd "${MODULE_DIR}" || perror "cd to '${MODULE_DIR}' failed."
+			copy_system_files
+		fi
 	fi
+	# Always copy static data
 	if [ -d "${MODULE_DIR}/data" ]; then
 		pinfo "## Copying static module files"
 		copy_static_data
@@ -377,6 +421,16 @@ process_module() {
 	pinfo "## Post copy"
 	cd "${MODULE_DIR}" || perror "cd to '${MODULE_DIR}' failed."
 	post_copy
+	# Write new md5 file if not existent yet
+	if [ ! -f "$MD5FILE" ]; then
+		md5sum "$MODULE_DIR/$MODULE".* > "$MD5FILE" || perror "Could not create $MD5FILE"
+	fi
+	# Update size of target build dir
+	if [ -d "${TARGET_BUILD_DIR}" ]; then
+		TARGET_BUILD_SIZE=$(du -bc "${TARGET_BUILD_DIR}" | awk 'END {print $1}')
+	else
+		TARGET_BUILD_SIZE=0
+	fi
 	# Sanity checks
 	[ -e "$TARGET_BUILD_DIR/var/run" -a ! -L "$TARGET_BUILD_DIR/var/run" ] && perror "Messup datected: $TARGET_BUILD_DIR/var/run exists and is not a symlink!"
 	[ -e "$TARGET_BUILD_DIR/var/lock" -a ! -L "$TARGET_BUILD_DIR/var/lock" ] && perror "Messup datected: $TARGET_BUILD_DIR/var/lock exists and is not a symlink!"
@@ -428,7 +482,7 @@ clean_modules() {
 	cd "$TARGET_DIR"
 
 	while (( "$#" )); do
-		clean_module $(readlink -f $1)
+		clean_module $(readlink -f "$1")
 		shift
 	done
 	cd - &> /dev/null
@@ -438,29 +492,20 @@ clean_modules() {
 }
 
 clean_module() {
+	[ -z "$1" ] && perror "No module given on clean_module()"
+	[ ! -d "$1" ] && perror "Module dir $1 doesn't exist!"
 	# if kernel is to be cleaned, do it separately and return
 	[ "x$(basename $1)" == "xkernel" ] && clean_kernel_module && return
 
 	pinfo "Cleaning '$1'..."
-	local MODULE_DIR=$1
-	if [ -e ${MODULE_DIR}/.built ]; then
-		rm "${MODULE_DIR}/.built" || perror "Could not clear built flag"
-	fi
-	if [ -e ${MODULE_DIR}/.fetched_source ]; then
-		rm "${MODULE_DIR}/.fetched_source" || perror "Could not clear fetched_source flag"
-	fi
-	if [ -d ${MODULE_DIR}/build ]; then
-		rm -rf "${MODULE_DIR}/build" || perror "Could not delete build path"
-	fi
-	if [ -d ${MODULE_DIR}/src ]; then
-		rm -rf "${MODULE_DIR}/src" || perror "Could not delete src path"
-	fi
-	if [ -e ${MODULE_DIR}/list_dpkg_output ]; then
-		rm "${MODULE_DIR}/list_dpkg_output" || perror "Could not delete list_dpkg_output"
-	fi
-	if [ -e ${MODULE_DIR}/list_binaries_and_files ]; then
-		rm "${MODULE_DIR}/list_binaries_and_files" || perror "Could not delete list_binaries_and_files"
-	fi
+	local MODULE_DIR="$1"
+	rm -rf -- "${MODULE_DIR}/build" || perror "Could not delete build path"
+	rm -rf -- "${MODULE_DIR}/src" || perror "Could not delete src path"
+	rm -f -- "${MODULE_DIR}/list_dpkg_output" || perror "Could not delete list_dpkg_output"
+	rm -f -- "${MODULE_DIR}/list_binaries_and_files" || perror "Could not delete list_binaries_and_files"
+	# These are not in use anymore, but leave the cleanup here for upgraders
+	rm -f -- "${MODULE_DIR}/.built"
+	rm -f -- "${MODULE_DIR}/.fetched_source"
 }
 
 clean_kernel_module() {