KERNEL_VERSION from system.inc

gitignore and calc_size in fileutil calculates the build size of a module

fix generate_target argument parsing

add xfs to rootfs-stage31.conf

added server sync option -s

- Fix sshd module failing to set permissions on sshd config
- Add all required packages for openSUSE to xorg module
- Softlink sh to bash in rootfs-stage32, as some scripts might fail otherwise

Thanks hwinfo but we don't need you anymore

Added size log

fix list_packet_files exiting loop when a packet wasnt installed

remove flag checks, now done in setup_target

check if kernel module is already built-in

6 files changed, 203 insertions, 22 deletions

diff --git a/helper/binutil.inc b/helper/binutil.inc
index b34230de..607041f0 100644
--- a/helper/binutil.inc
+++ b/helper/binutil.inc
@@ -23,20 +23,16 @@ get_dynamic_dependencies() {
 	
 	if [ "x$1" == "x-l" ]; then
 		LOCALSEARCH=1
-		pdebug "Local search activated."
 		shift
 		[ ! -d $1 ] && perror "Directory given does not exist, exiting."
 		LOCALSEARCHDIR=$1
 		shift
-		pdebug "Looking in: $LOCALSEARCHDIR"
 	fi	
 
 	while [ $# != 0 ]; do
 		local BINARY=$1
 		shift
 
-		pdebug "Processing $BINARY ..."
-		
 		local LDD_OUT="ldd_output"
 		if ldd $BINARY > $LDD_OUT; then
 			# Case 1: dynamic
@@ -49,7 +45,7 @@ get_dynamic_dependencies() {
 			done
 		else
 			# Case 2: not a dynamic
-			pdebug "$BINARY not a dynamic, skipping."
+			pdebug "\t\t\t(Not a dynamic.)"
 			rm -f $LDD_OUT
 			continue
 		fi	
@@ -60,7 +56,6 @@ get_dynamic_dependencies() {
 
 lib_search(){
 
-	pdebug "\tSearching for ${liblink[0]}..."
 	# if activated, start by searching the lib locally
 	if [ "x$LOCALSEARCH" == "x1" ]; then
 		cd $LOCALSEARCHDIR
@@ -68,7 +63,6 @@ lib_search(){
 		cd - &>/dev/null
 	        if [ "x${LOCAL_MATCHES}" != "x" ]; then
         		for llib in ${LOCAL_MATCHES}; do
-	        	        pdebug "\t\tFound locally, copying ${LOCALSEARCHDIR}/${llib}"
 				get_link_chain "${LOCALSEARCHDIR}"/"${llib}" "${LOCALSEARCHDIR}"
 				get_dynamic_dependencies -l "${LOCALSEARCHDIR}" "${llib}"
 				CURRENT_BLACKLIST+="\|${liblink[0]}"
@@ -81,7 +75,6 @@ lib_search(){
 
 	# search the lib on the system since it was not found earlier
 	if [ ! -z ${liblink[1]} ] && [ "x${liblink[1]}" != "xnot" ]; then
-		pdebug "\t\tNot found locally but in system, copying ${liblink[1]}"
 		# get chain of symlink for that lib
                 get_link_chain ${liblink[1]}
 		CURRENT_BLACKLIST+="\|${liblink[1]}"
@@ -161,7 +154,7 @@ get_link_chain() {
 	                echo "$LINK"
         	fi
 	done
-	pdebug "\t\tCHAIN: $CHAIN"
+	pdebug "\t\t$CHAIN"
 }
 
 # Function to get libc and ld-linux
diff --git a/helper/downloader.inc b/helper/downloader.inc
index 7ded5b4f..e12c8a02 100644
--- a/helper/downloader.inc
+++ b/helper/downloader.inc
@@ -6,7 +6,9 @@
 # 2. download "http://example.com/something.tar.gz" "somename.tar.gz"
 download () {
 	[ $# -lt 1 -o $# -gt 2 ] && perror "download called with $# arguments, need 1 or 2"
+	[ -z "$1" ] && perror "download: URL empty."
 	if [ $# -eq 2 ]; then
+		[ -z "$2" ] && perror "download: target file name given but empty"
 		pinfo "Downloading $2 from '$1'...."
 		wget -O "$2" "$1"
 		local RET=$?
@@ -24,13 +26,14 @@ download () {
 # 2. download_untar "http://example.com/something.tar.gz" "src/" "temporary_name.tar.gz"
 download_untar () {
 	[ $# -lt 2 -o $# -gt 3 ] && perror "download_untar called with $# arguments, need 2 or 3"
-	local URL=$1
-	local DEST=$2
+	local URL="$1"
+	local DEST="$2"
 	if [ $# -eq 2 ]; then
 		local TMPFILE=dltmp.$(basename "$URL")
 	else
-		local TMPFILE=$3
+		local TMPFILE="$3"
 	fi
+	pdebug "$URL ---> $TMPFILE"
 	download "$URL" "$TMPFILE"
 	mkdir -p "$DEST"
 	pinfo "Unpacking to '$DEST'..."
@@ -40,3 +43,29 @@ download_untar () {
 	unlink "$TMPFILE"
 }
 
+# Download first param URL to second param path,
+# iff the local file does not exist or is empty.
+# Return 1 if remote file does not exist, 0 otherwise
+download_if_empty() {
+	[ $# -ne 2 ] && perror "download_if_empty: want 2 args, got $# ($@)"
+	local SRC="$1"
+	local DST="$2"
+	[ -s "$DST" ] && pdebug "Not downloading $DST: already there." && return 0
+	pdebug "Downloading $DST"
+	[ -e "$DST" ] && unlink "$DST"
+	local DSTDIR="$(dirname "$DST")"
+	pdebug "wgetting $SRC"
+	local TMP=$(mktemp)
+	wget -O "$TMP" "$SRC"
+	local RET=$?
+	if [ "x$RET" = "x0" -a -s "$TMP" ]; then
+		mkdir -p "$DSTDIR"
+		mv "$TMP" "$DST" || perror "Could not mv '$TMP' '$DST'"
+		return 0
+	fi
+	pdebug "WGET failed"
+	rmdir "$DSTDIR"
+	unlink "$TMP"
+	return 1
+}
+
diff --git a/helper/fileutil.inc b/helper/fileutil.inc
index d39fbe87..07dab7af 100755..100644
--- a/helper/fileutil.inc
+++ b/helper/fileutil.inc
@@ -24,7 +24,8 @@ tarcopy () {
 	[ ${#SHORT} -gt 30 ] && SHORT=$(echo "$SHORT" | sed ':a;N;$!ba;s/\n/ /g' | cut -c-25)...$(echo "$SHORT" | cut -c$[${#SHORT} - 4]-)
 	[ -z "$TO" ] && perror "tarcopy called with empty destination."
 	[ ! -d "$TO" ] && { mkdir -p "$TO" || perror "could not create destination "$TO" for tar-copy."; } 
-	tar -cp $FROM | tar -xp -C "$TO"
+	# TODO count files copied? would remove the need to do it everywhere :)
+	tar -cpP $FROM | tar -xp -C "$TO" 2> /dev/null
 	local PS=(${PIPESTATUS[*]})
 	[ "x${PS[0]}" != "x0" ] && perror "packing-part of tar-copy from '$SHORT' to '$TO' failed. (${PS[0]})"
 	[ "x${PS[1]}" != "x0" ] && perror "unpacking-part of tar-copy from '$SHORT' to '$TO' failed. (${PS[1]})"
@@ -45,8 +46,8 @@ list_packet_files() {
 		#FILES=$(echo "$FILES" | sed 's/^\(.*\):###:[0-9]*$/\1/g')
 		local LPRET=$(echo "$FILES" | awk -F ':###:' '{printf $2}')
 		FILES=$(echo "$FILES" | awk -F ':###:' '{print $1}')
-		[ "x$LPRET" != "x0" ] && pwarning "dpkg/rpm exited with code '$LPRET' for packet ${PACKAGE}." && return 1
-		[ -z "$FILES" ] && pwarning "list_packet_files empty for packet ${PACKAGE}."
+		[ "x$LPRET" != "x0" ] && pwarning "dpkg/rpm exited with code '$LPRET' for packet ${PACKAGE}." && continue
+		[ -z "$FILES" ] && pwarning "list_packet_files empty for packet ${PACKAGE}." && continue
 		for FILE in $FILES; do
 			[ ! -d "$FILE" ] && echo "$FILE"
 		done
@@ -56,8 +57,10 @@ list_packet_files() {
 # install all dependencies of a module
 # goes through all package as given in the variable REQUIRED_DEPENDENCIES
 install_dependencies() {
-	[ -z "$REQUIRED_DEPENDENCIES" ] && return
+	[ -z "$REQUIRED_DEPENDENCIES" ] && return 
 	if [ "$PACKET_MANAGER" = "apt" ]; then
+		# install package, only if not already installed
+		dpkg -l $(echo $(echo "$REQUIRED_DEPENDENCIES")) &> /dev/null && return
 		apt-get install -y $REQUIRED_DEPENDENCIES || perror "Could not apt-get install $REQUIRED_DEPENDENCIES"
 	elif [ "$PACKET_MANAGER" = "zypper" ]; then
 		zypper --no-refresh install -y -n $REQUIRED_DEPENDENCIES || perror "Could not zypper install $REQUIRED_DEPENDENCIES"
@@ -68,11 +71,30 @@ install_dependencies() {
 #
 copy_static_data() {
 	[ ! -d "${MODULE_DIR}/data" ] && pinfo "${MODULE} has no static 'data' directory." && return
-	cp -r ${MODULE_DIR}/data/* ${TARGET_BUILD_DIR} || perror "Could not copy static data of ${MODULE}"
+	cp -r "${MODULE_DIR}/data/"* ${TARGET_BUILD_DIR} || perror "Could not copy static data of ${MODULE}"
 }
 
+calc_size() {
+
+	local CURRENT_BUILD_SIZE=$(du -bc "${TARGET_BUILD_DIR}" | awk 'END {print $1}')
+
+	[ ! -z "${BUILD_SIZE[$MODULE]}" ] && local OLD_MODULE_SIZE=${BUILD_SIZE[$MODULE]} || local OLD_MODULE_SIZE=0
+	local diff=$((CURRENT_BUILD_SIZE-TARGET_BUILD_SIZE+OLD_MODULE_SIZE))
+
+	if [ -z "${BUILD_SIZE[$MODULE]}" ]; then 
+		echo "BUILD_SIZE[$MODULE]=${diff}" >> "${ROOT_DIR}/logs/${TARGET}.size"
+	else
+		sed -i "s/^BUILD_SIZE\[${MODULE}\]=.*$/BUILD_SIZE\[${MODULE}\]=${diff}/g" "${ROOT_DIR}/logs/${TARGET}.size"
+	fi
+	
+	MODULE_BUILD_SIZE=$(echo $diff | awk '{ sum=$1; hum[1024^3]="GB"; hum[1024^2]="MB"; hum[1024]="KB"; 
+					for (x=1024^3; x>=1024; x/=1024){ 
+        					if (sum>=x) { printf "%.2f %s\n",sum/x,hum[x]; break }
+					}
+				}')
+	TARGET_BUILD_SIZE=$CURRENT_BUILD_SIZE
+}
 
-######################################################################################################################
 #
 # generate initramfs of directory
 # usage:
@@ -80,7 +102,6 @@ copy_static_data() {
 # example:
 #	generate_initramfs "./server/boot/stage32_sqfs" "./mnt/openslx.sqfs" "./server/boot/initramfs2"
 #	generate_initramfs "./server/build/stage31" "." "./server/boot/initramfs"
- 
 generate_initramfs() {
 	[ $# -ne 3 ] && perror "Sanity check failed: generate_initramfs needs exactly two params, but $# were given."
         cd "$1" || perror "Cannot cd to '$1'"
@@ -90,7 +111,7 @@ generate_initramfs() {
         [ "x${PS[0]}" != "x0" ] && perror "'find $2' in '$(pwd)' failed."
         [ "x${PS[1]}" != "x0" ] && perror "cpio create failed."
         [ "x${PS[2]}" != "x0" ] && perror "gzip to '$3' failed."
-        cd -
+        cd - &> /dev/null
         pinfo "Created initramfs of $1 at $3"
 }
 
diff --git a/helper/kernel.inc b/helper/kernel.inc
new file mode 100644
index 00000000..4216f25a
--- /dev/null
+++ b/helper/kernel.inc
@@ -0,0 +1,137 @@
+# 
+#	Common functions to copy kernel related files
+#
+############################################################
+#
+#
+# copies kernel modules as given in the module config file
+# * depends on 'depmod'
+# * requires REQUIRED_KERNEL_MODULES to be set.
+#   (entries must be a relative path to /lib/modules/<KERNEL_VERSION>)
+#
+# ex: for 		/lib/modules/3.2.0/kernel/fs/nfs/nfs.ko
+#     must be given as  kernel/fs/nfs/nfs.ko
+#
+
+# this code depends on KERNEL_VERSION, this file needs to be sourced after helper/system.inc!
+# (TODO: maybe source system.inc if KERNEL_VERSION is empty.
+#[ -z "${KERNEL_VERSION}" ] && perror "KERNEL_VERSION not set. Was helper/system.inc sourced?"
+
+KERNEL_NFS_DIR=""
+mount_kernel_dir() {
+	[ -z "$KERNEL_NFS_DIR" ] || return 0
+	KERNEL_NFS_DIR="$(mktemp -d)"
+	mount -t nfs -o ro "132.230.8.228:/srv/openslx/kernel" "$KERNEL_NFS_DIR"
+	local RET=$?
+	if [ "x$RET" != "x0" ]; then
+		KERNEL_NFS_DIR=""
+		perror "Mounting kernel nfs dir to $KERNEL_NFS_DIR failed."
+		exit 1
+	fi
+	#pinfo "$KERNEL_NFS_DIR/$SELECTED_KERNEL"
+	#qnd_exit
+	[ -d "$KERNEL_NFS_DIR/$SELECTED_KERNEL" ] || perror "directory for $KERNEL_VERSION ($SELECTED_KERNEL) does not exist on NFS server"
+}
+
+unmount_kernel_dir() {
+	[ -z "$KERNEL_NFS_DIR" ] && return 0
+	pinfo "Unmounting kernel nfs stuff....."
+	umount "$KERNEL_NFS_DIR" || perror "Could not unmount kernel NFS share at '$KERNEL_NFS_DIR' - check the following lsof output:\n$(lsof -n | grep "$KERNEL_NFS_DIR")\n- End of lsof output -"
+	rmdir "$KERNEL_NFS_DIR"
+	KERNEL_NFS_DIR=""
+}
+
+copy_kernel_modules() {
+	
+	[ -z "${REQUIRED_KERNEL_MODULES}" ] && perror "REQUIRED_KERNEL_MODULES is empty. Check your config file."
+	
+	#
+        # process modules list
+	#
+	mount_kernel_dir
+	cd "$KERNEL_NFS_DIR/$SELECTED_KERNEL" || perror "Could not cd to $KERNEL_NFS_DIR/$SELECTED_KERNEL"
+	local KERNEL_MODULES_DIR="lib/modules/${KERNEL_VERSION}"
+	local KERNEL_MODULES_LIST=""
+        for KERNEL_MODULE in ${REQUIRED_KERNEL_MODULES}; do
+                local KERNEL_MODULE_PATH="${KERNEL_MODULES_DIR}/${KERNEL_MODULE}"
+		if grep "${KERNEL_MODULE}" "${KERNEL_NFS_DIR}/${SELECTED_KERNEL}/${KERNEL_MODULES_DIR}/modules.builtin" >/dev/null; then
+			pdebug "Already built-in ${KERNEL_MODULE}."
+                elif [ -e "${KERNEL_MODULE_PATH}" ]; then
+                        pdebug "Copying '${KERNEL_MODULE_PATH}'"
+                        MODULES_LIST+=" ${KERNEL_MODULE_PATH}"
+		else
+                        pwarning "Module ${KERNEL_MODULE} not found. Skipping. (might cause problems on certain clients!)"
+                        continue
+                fi
+        done
+
+        if [ ! -z "${KERNEL_MODULES_LIST}" ]; then
+                local COUNT=$(echo "${KERNEL_MODULES_LIST}" | wc -w)
+                pinfo "Copying $COUNT modules to target directory."
+                tarcopy "${KERNEL_MODULES_LIST}" "${TARGET_BUILD_DIR}"
+        fi
+
+	#
+	# generate modules map files
+	#
+        # first strip modules.order of all the modules we don't use
+        cat "${KERNEL_MODULES_DIR}/modules.order" | grep -E $(echo ${REQUIRED_KERNEL_MODULES} | tr '\ ' '|') \
+                >> "${TARGET_BUILD_DIR}/${KERNEL_MODULES_DIR}/modules.order"
+        # copy list of builtin kernel modules
+        cp "${KERNEL_MODULES_DIR}/modules.builtin" "${TARGET_BUILD_DIR}/${KERNEL_MODULES_DIR}"
+        # with modules.order and modules.builtin, we can run depmod for the rest of the files
+        depmod -b "${TARGET_BUILD_DIR}" -a "${KERNEL_VERSION}"
+
+	cd - >/dev/null
+}
+
+copy_firmware() {
+
+	[ -z "${REQUIRED_FIRMWARE}" ] && perror "REQUIRED_FIRMWARE is empty. Check your config file."
+
+	#
+        # process firmware list
+	#
+	mount_kernel_dir
+	cd "$KERNEL_NFS_DIR/$SELECTED_KERNEL" || perror "Could not cd to $KERNEL_NFS_DIR/$SELECTED_KERNEL"
+        local FIRMWARE_DIR="lib/firmware"
+	local FIRMWARE_LIST=""
+        for FIRMWARE in ${REQUIRED_FIRMWARE}; do
+                local FOUND=0
+		for CANDIDATE in "${FIRMWARE_DIR}/${FIRMWARE}" "${FIRMWARE_DIR}/${KERNEL_VERSION}/${FIRMWARE}"; do
+			if [ -e "${CANDIDATE}" ]; then
+				pdebug "Copying '${CANDIDATE}'"
+				FIRMWARE_LIST+=" ${CANDIDATE}"
+				FOUND=1
+			fi
+		done
+
+                [ $FOUND -ne 1 ] && pwarning "Neither '${FIRMWARE_DIR}/${FIRMWARE}' nor '${FIRMWARE_DIR}/${KERNEL_VERSION}/${FIRMWARE}' "\
+						" was found on the system. Skipping. (might cause problems on certain clients!)"
+        done
+
+        if [ ! -z "${FIRMWARE_LIST}" ]; then
+                local COUNT=$(echo "${FIRMWARE_LIST}" | wc -w)
+                pinfo "Copying $COUNT firmware to target directory."
+                tarcopy "${FIRMWARE_LIST}" "${TARGET_BUILD_DIR}"
+        fi
+	cd - >/dev/null
+}
+
+copy_kernel() {
+	
+	local KERNEL_NAME="vmlinuz-${KERNEL_VERSION}"
+	[ -e "${KERNEL_DIR}/${KERNEL_NAME}" ] && return 
+	
+       	local TOOL_STR="$TOOL_STR copy_kernel:"
+
+	mkdir -p "${KERNEL_DIR}"
+
+	pinfo "New kernel found. Copying '${KERNEL_NAME}' to '${KERNEL_DIR}'."
+	mount_kernel_dir
+
+	cp "$KERNEL_NFS_DIR/$SELECTED_KERNEL/bzImage" "$KERNEL_DIR/$KERNEL_NAME" || perror "Could not copy kernel from '$KERNEL_NFS_DIR/$SELECTED_KERNEL/bzImage' to '$KERNEL_DIR/$KERNEL_NAME'"
+
+	pinfo "You may want to update your systems firmware/modules to match the current kernel."
+}
+
diff --git a/helper/logging.inc b/helper/logging.inc
index eb20775c..a237edc2 100644
--- a/helper/logging.inc
+++ b/helper/logging.inc
@@ -34,6 +34,7 @@ pinfo  () {
 
 perror () { 
 	echo -e "\033[38;5;9m[error]\033[0m   $TOOL_STR $@" >&6
+	unmount_kernel_dir
 	qnd_exit
 }
 
diff --git a/helper/system.inc b/helper/system.inc
index 4c0ea467..21e92422 100644
--- a/helper/system.inc
+++ b/helper/system.inc
@@ -11,8 +11,8 @@ ARCH_LIB_DIR=$(ldd $SHELL | grep "libc.so" | sed -r 's#^.*(/lib.*)/libc.so.*$#\1
 [ -z "$ARCH_LIB_DIR" -o ! -d "$ARCH_LIB_DIR" ] && perror "Could not determine arch dependent lib dir (where libc.so resides)"
 
 # determine kernel version
-KERNEL_VERSION=$(uname -r)
-[ ! -z $KERNEL_VERSION ] || pwarning "Could not determine kernel version."
+#KERNEL_VERSION=$(uname -r)
+#[ ! -z $KERNEL_VERSION ] || pwarning "Could not determine kernel version."
 
 # determine number of CPU cores
 CPU_CORES=$(cat /proc/cpuinfo | grep processor | wc -l)