#!/bin/bash
# -----------------------------------------------------------------------------
#
# Copyright (c) 2011 - OpenSLX GmbH
#
# This program is free software distributed under the GPL version 2.
# See http://openslx.org/COPYING
#
# If you have any feedback please consult http://openslx.org/feedback and
# send your suggestions, praise, or complaints to feedback@openslx.org
#
# General information about OpenSLX can be found at http://openslx.org/
# -----------------------------------------------------------------------------
#
# This file contains the functions needed to setup a target.
# The main function is 'generate_target' which will go over all
# activated modules (symlinks found under remote/targets/<target>)
# and run the module-specific 'fetch_source', 'build' and 'post_copy'
# functions of the modules.
#
# The functions defined here are module-independent, meaning they
# can process any module, as long as following requirements are met:
#
# - Modules need to have a build-script and a config file,
# i.e. remote/module/kernel/kernel.build
# and remote/module/kernel/kernel.conf
# as these will be read by 'process_module'.
#
# - Modules need to generate a 'build'-directory in their
# own sub-directory (i.e. remote/modules/sshd/build/
# as 'copy_files_with_deps' will look for the files in
# that directory ONLY.
#
# - Several information have to be set in the config file of
# a module:
# * REQUIRED_BINARIES - list of binaries
# * REQUIRED_FILES - list of regular files
# * REQUIRED_DIRECTORIES - list of directories
# * REQUIRED_SYSTEM_FILES - files to be copied directly from the system
# * REQUIRED_INSTALLED_PACKAGES - list of packages needed at build-time
# * REQUIRED_CONTENT_PACKAGES - list of packages where the contents are to be copied
# * (optional variables needed by the module-build-script)
#
# NOTE: REQUIRED_BINARIES, REQUIRED_FILES, REQUIRED_FILES will be ONLY looked
# for in the module's own build directory!
#
# Optionally, a module can have a static data directory under
# remote/modules/<module>/data which will be copied as is to the target build directory.
#
# Moreover modules do not need to specify the dynamic dependencies of given binaries
# as these will be determined automaticly by the helper function 'get_dynamic_dependencies'
# -----------------------------------------------------------------------------
#
MODE_DIR="${ROOT_DIR}/remote"
MODULES_DIR="${MODE_DIR}/modules"
EXPORT_DIR="/export/build"
# Keep track of processed modules
PROCESSED_MODULES=""
initial_checks () {
:
}
#
# generic function to read the config file of the current $MODULE
#
read_config () {
# unset previous variables from other config files
for VARNAME in ${!REQUIRED_*}; do
unset $VARNAME
done
local MODULE_CONFIG="${MODULE_DIR}/${MODULE}.conf"
# sanity checks
[ ! -e "${MODULE_CONFIG}" ] && perror "Config for '$MODULE' not found."
# source the config file
. "${MODULE_CONFIG}" || perror "Sourcing '${MODULE_CONFIG}' failed."
if [ -e "${MODULE_CONFIG}.${PACKET_MANAGER}" ]; then
# a specific tool.conf seems to exist, use it to override certain vars
. "${MODULE_CONFIG}.${PACKET_MANAGER}" || perror "Sourcing '${MODULE_CONFIG}.${PACKET_MANAGER}' failed."
fi
}
#
# generic function to read the build script of the current $MODULE
#
read_build () {
local BUILD_SCRIPT="${MODULE_DIR}/${MODULE}.build"
[ ! -e "${BUILD_SCRIPT}" ] && perror "Build script for specified tool not found."
. "${BUILD_SCRIPT}" || perror "Sourcing '${BUILD_SCRIPT}' failed."
}
export_builds() {
[ ! -d "${MODE_DIR}/builds" ] && perror "No ${MODE_DIR}/builds, nothing to export."
pinfo "Mounting ${MODE_DIR}/builds to ${EXPORT_DIR}. This will make the local builds syncable from another machine."
[ ! -d "${EXPORT_DIR}" ] && mkdir -p "${EXPORT_DIR}"
mount --bind ${MODE_DIR}/builds ${EXPORT_DIR} || perror "Failed to bind mount ${MODE_DIR}/builds to ${EXPORT_DIR}"
}
#
# main function which copies all files, directories, binaries and external libraries to TARGET_BUILD_DIR
#
copy_files_with_deps () {
[ ! -d "$MODULE_BUILD_DIR" ] && pinfo "No build directory found, skipping dependency copying" && return 0
cd "$MODULE_BUILD_DIR"
COPYFILES_LIST="list_wanted_stage3.2"
[ -e "${COPYFILES_LIST}" ] && rm "${COPYFILES_LIST}"
[ ! -z "${REQUIRED_BINARIES}" ] && pinfo "Gathering required binaries from config file..."
for FILENAME in ${REQUIRED_BINARIES}
do
local FILE_CANDIDATES=$( find . -name "${FILENAME}" -a \( -type f -o -type l \) )
# only do if more than one candidate found
if [ "$(echo $FILE_CANDIDATES | wc -w)" -gt 1 ]; then
pdebug "Candidates for $FILENAME are: $(echo $FILE_CANDIDATES)"
local FINAL_LIST=""
for FILE in $FILE_CANDIDATES; do
local TESTFILE="$(readlink -f "$FILE")"
pdebug " $FILE leads to $TESTFILE"
[ -f "$TESTFILE" -a -x "$TESTFILE" ] && [ "x$(grep -l -E '^(.ELF|#!)' "$TESTFILE")" != "x" ] && FINAL_LIST="$FINAL_LIST $FILE"
done
FINAL_LIST=$(trim "$FINAL_LIST")
if [ -z "$FINAL_LIST" ]; then
pwarning "\tNo Binary found for ${FILENAME}. Skipping."
continue
fi
if [[ "$FINAL_LIST" == *" "* ]]; then
pwarning "Found more than one match for required file '$FILENAME': $FINAL_LIST"
else
pdebug "\tFound ${FILENAME} at ${FILE}"
fi
elif [ "$(echo $FILE_CANDIDATES | wc -w)" -eq 0 ]; then
perror "Could not find required binary $FILENAME"
else
# one candidate
FINAL_LIST=${FILE_CANDIDATES}
fi
for FILE in $FINAL_LIST; do
pdebug "* $FILE"
strip "$FILE" || pwarning "Could not strip '${FILE}'"
get_link_chain "${MODULE_BUILD_DIR}/${FILE}" "${MODULE_BUILD_DIR}" >> "${COPYFILES_LIST}"
get_dynamic_dependencies -l "${MODULE_BUILD_DIR}" "${FILE}" >> "${COPYFILES_LIST}"
done
done
[ ! -z "${REQUIRED_LIBRARIES}" ] && pinfo "Gathering required libraries from config file..."
for LIB in ${REQUIRED_LIBRARIES}
do
for LOCATION in $(find . -name ${LIB}.so\*)
do
pdebug "* $LOCATION"
strip "$LOCATION" || pwarning "Could not strip '${LOCATION}'"
get_link_chain "${MODULE_BUILD_DIR}/${LOCATION}" "${MODULE_BUILD_DIR}" >> "${COPYFILES_LIST}"
get_dynamic_dependencies -l "${MODULE_BUILD_DIR}" "${LOCATION}" >> "${COPYFILES_LIST}"
done
done
[ ! -z "${REQUIRED_DIRECTORIES}" ] && pinfo "Gathering required directories from config file..."
local CURRENT_PWD=$(pwd) # Prevent calling pwd 50000 times inside the loop below
for ENTRY in ${REQUIRED_DIRECTORIES}
do
pdebug "* .$ENTRY"
echo "./${ENTRY}" >> "${COPYFILES_LIST}"
for BIN in $(find ".${ENTRY}" -type f -not -name '*.a' | xargs grep -l '^.ELF')
do
#pdebug "\tSearching libs for ${BIN}..."
get_link_chain "${MODULE_BUILD_DIR}/${BIN}" "${MODULE_BUILD_DIR}" >> "${COPYFILES_LIST}"
get_dynamic_dependencies -l "${MODULE_BUILD_DIR}" "${BIN}" >> "${COPYFILES_LIST}"
done
done
[ ! -z "${REQUIRED_FILES}" ] && pinfo "Gathering required files from config file..."
for ENTRY in ${REQUIRED_FILES}
do
get_link_chain "${MODULE_BUILD_DIR}/${ENTRY}" "${MODULE_BUILD_DIR}" >> "${COPYFILES_LIST}"
get_dynamic_dependencies -l "${MODULE_BUILD_DIR}" "${MODULE_BUILD_DIR}/.${ENTRY}" >> "${COPYFILES_LIST}"
done
#copy to initramfsdir
pdebug "File list generated at ${MODULE_BUILD_DIR}/${COPYFILES_LIST}."
if [ -s "$COPYFILES_LIST" ]; then
local CLISTCOUNT=$(cat "$COPYFILES_LIST" | wc -l)
pinfo "Copying $CLISTCOUNT files to '${TARGET_BUILD_DIR}'."
tarcopy "$(cat "$COPYFILES_LIST"|sort -u)" "${TARGET_BUILD_DIR}"
fi
}
#
#
# main public function. Requires the TARGET to be given as argument.
# this will simply go over all the modules as found in the remote/target/<TARGET>
# and run following functions:
#
#
generate_target() {
initial_checks
TARGET=$1 && shift
TARGET_DIR="${MODE_DIR}/targets/${TARGET}"
TARGET_BUILD_DIR="${MODE_DIR}/builds/${TARGET}"
[ -e "${ROOT_DIR}/logs/${TARGET}.size" ] && . "${ROOT_DIR}/logs/${TARGET}.size" || echo "declare -A BUILD_SIZE" >> "${ROOT_DIR}/logs/${TARGET}.size"
[ -d "$TARGET_DIR" ] || perror "Given target directory does not exist: $TARGET_DIR"
[[ "$TARGET" == "builds" || "$TARGET" == "modules" ]] && \
perror "Target directory cannot be named 'builds' or 'modules'."
pinfo "Generating '$TARGET_BUILD_DIR' for '$TARGET'"
# if no arguments assume all.
if [ "x$1" = "x" -o "x$1" = "xall" ]; then
MODULES=$(ls ${TARGET_DIR})
set -- $MODULES
else
# tools = arguments given
MODULES=$@
fi
pinfo "Activated modules in '${TARGET}':"
pinfo "\t$(echo ${MODULES})"
# copy basic libs
pinfo "Copying libc and ld-linux used by ${SHELL}"
tarcopy "$(list_basic_libs)" "${TARGET_BUILD_DIR}"
[ -d "$TARGET_BUILD_DIR" ] && TARGET_BUILD_SIZE=$(du -bc "${TARGET_BUILD_DIR}" | awk 'END {print $1}') || TARGET_BUILD_SIZE=0
# now iterate over given tools and copy them
while (( "$#" )); do
process_module "$1"
shift
done
post_process_target
pinfo "Target completed. Total size: $(du -bsh "${TARGET_BUILD_DIR}" | awk 'END {print $1}')"
TOOL_STR=""
}
process_module() {
[ "$#" -ne "1" ] && perror "process_module: want 1 param."
local MODULE="$1"
[[ "$PROCESSED_MODULES" == *"!${MODULE}!"* ]] && return # Already processed this module
PROCESSED_MODULES="${PROCESSED_MODULES}!${MODULE}!"
local MODULE_DIR="${TARGET_DIR}/${MODULE}"
local MODULE_BUILD_DIR="${MODULE_DIR}/build"
local TOOL_STR=""
pinfo ">>>>>>>>>>>>>>>>> Processing module [ $MODULE ]"
TOOL_STR="[${MODULE}]"
if [ -d "${MODULE_DIR}" ]; then
#[ "x$DEBUG" != "x1" ] \
# && echo "Logging to ${TOOL_DIR}/${TOOL}/stage32.log" \
# && exec 6>&1 > ${TOOL_DIR}/${TOOL}/stage32.log
# TODO: Make above work with the new logging system (add function to logging.inc to switch logfile)
cd "${MODULE_DIR}" || perror "Module dir '${MODULE_DIR}' seems to exist, but cd to it failed."
pinfo "## Reading config"
read_config
# Check if this module has a dependency that wasn't built yet:
if [ ! -z "$REQUIRED_MODULES" ]; then
pinfo "$MODULE depends on ${REQUIRED_MODULES}...."
for DEP in $REQUIRED_MODULES; do
process_module "$DEP"
done
# Read old config again, as it got overwritten by the deps
cd "${MODULE_DIR}" || perror "Tool dir '${MODULE_DIR}' seems to exist, but cd to it failed (after building deps)."
read_config
pinfo "<<<<<<<<<<<<<<<<< Dependency modules processed, back to module [ $MODULE ]"
fi
pinfo "## Reading build"
read_build
pinfo "## Installing dependencies"
install_dependencies
pinfo "## Fetching source"
[ -e "${MODULE_DIR}/.fetched_source" ] || { fetch_source && touch "${MODULE_DIR}/.fetched_source"; }
if [ -e "${MODULE_DIR}/.built" -a -d "${MODULE_BUILD_DIR}" ]; then
pinfo "# built-flag is set, skipping build..."
else
pinfo "## Building"
mkdir -p "${MODULE_BUILD_DIR}" || perror "Could not create build dir"
build # calls perror if something fails, no need to do that here
touch "${MODULE_DIR}/.built" || pwarning "Error setting built-flag"
fi
# remove *.la files as they might confuse libtool/linker of other tool packages
[ -d "${MODULE_BUILD_DIR}" ] && find "${MODULE_BUILD_DIR}" -name '*.la' -exec rm -f {} \;
[ -d "${TARGET_BUILD_DIR}" ] && TARGET_BUILD_SIZE=$(du -bc "${TARGET_BUILD_DIR}" | awk 'END {print $1}') || TARGET_BUILD_SIZE=0
pinfo "## Copying files with dependencies"
copy_files_with_deps
pinfo "## Copying required system files"
copy_system_files
pinfo "## Copying static module files"
copy_static_data
pinfo "## Post copy"
post_copy
# reset pipes
#[ "x$DEBUG" != "x1" ] && exec 1>&6 6>&-
# TODO
calc_size #sets MODULE_BUILD_SIZE
pinfo "Module completed. Total size: ${MODULE_BUILD_SIZE}"
else
pwarning "Module directory for '$MODULE' not found."
fi
}
post_process_target() {
local TOOL_STR="$TOOL_STR post_process_target:"
#move systemd files from /lib to /etc/lib
pinfo "move systemd files from /lib/systemd to /etc/systemd"
if [ -d "${TARGET_BUILD_DIR}/lib/systemd" ]; then
cd "${TARGET_BUILD_DIR}/lib"
tarcopy "systemd" "${TARGET_BUILD_DIR}/etc"
cd - &> /dev/null
rm -r "${TARGET_BUILD_DIR}/lib/systemd"
fi
# figure out all relevant ld-paths
local LDTEMP="$TARGET_BUILD_DIR/ldconf-temp"
grep -h '^\s*/' /etc/ld.so.conf /etc/ld.so.conf.d/* > "$LDTEMP"
# TODO: Echo other stuff to $LDTEMP (but figure out if first or last match wins)
ldconfig -v -r "${TARGET_BUILD_DIR}" -f "/ldconf-temp"
unlink "$LDTEMP"
}
clean_modules() {
TARGET=$1
shift
TARGET_DIR="${MODE_DIR}/targets/${TARGET}"
TARGET_BUILD_DIR="${MODE_DIR}/builds/${TARGET}"
[ -d $TARGET_DIR ] || perror "Given target directory does not exist: $TARGET_DIR"
if [ "x$1" = "x" -o "x$1" = "xall" ]; then
if [ -e ${ROOT_DIR}/logs/${TARGET}.size ]; then
rm "${ROOT_DIR}/logs/${TARGET}.size" || perror "Could not delete logs/${TARGET}.size"
fi
if [ -d ${TARGET_BUILD_DIR} ]; then
pinfo "Cleaning '${TARGET_BUILD_DIR}'" \
&& rm -rf "${TARGET_BUILD_DIR}"/* \
|| perror "Error deleting $TARGET_BUILD_DIR"
fi
set -- $(ls ${TARGET_DIR} | grep -v kernel)
fi
cd ${TARGET_DIR}
while (( "$#" )); do
clean_module $(readlink -f $1)
shift
done
cd - &> /dev/null
}
clean_module() {
# if kernel is to be cleaned, do it separately and return
[ "x$(basename $1)" == "xkernel" ] && clean_kernel_module && return
pinfo "Cleaning '$1'..."
local MODULE_DIR=$1
if [ -e ${MODULE_DIR}/.built ]; then
rm "${MODULE_DIR}/.built" || perror "Could not clear built flag"
fi
if [ -e ${MODULE_DIR}/.fetched_source ]; then
rm "${MODULE_DIR}/.fetched_source" || perror "Could not clear fetched_source flag"
fi
if [ -d ${MODULE_DIR}/build ]; then
rm -rf "${MODULE_DIR}/build" || perror "Could not delete build path"
fi
if [ -d ${MODULE_DIR}/src ]; then
rm -rf "${MODULE_DIR}/src" || perror "Could not delete src path"
fi
if [ -e ${MODULE_DIR}/list_dpkg_output ]; then
rm "${MODULE_DIR}/list_dpkg_output" || perror "Could not delete list_dpkg_output"
fi
if [ -e ${MODULE_DIR}/list_binaries_and_files ]; then
rm "${MODULE_DIR}/list_binaries_and_files" || perror "Could not delete list_binaries_and_files"
fi
}
clean_kernel_module() {
pinfo "Cleaning kernel module (including sources and compiled stuff)."
pinfo "It will take a long time to rebuild, are you sure? [y/N]"
read USER_INPUT
if [ "x$USER_INPUT" == "xy" ]; then
pinfo "Confirmed, going on..."
# clean kernel module
cd "${TARGET_DIR}/kernel" || perror "Could not cd to ${TARGET_DIR}/kernel"
if [ -e build ]; then
rm -rf build || perror "Could not delete ${TARGET_DIR}/kernel/build"
fi
# little workaround to actually list names of entries
# (if only a dir exists, ls kernel/linux-* would just list
# the contents of that directory....)
for FILE in $(ls | grep linux); do
rm -rf $FILE || perror "Could not delete $TARGET_DIR}/$FILE"
done
# finally clean the config generated
if [ -e openslx.config ]; then
rm openslx.config || perror "Could not delete ${TARGET_DIR}/kernel/openslx.config"
fi
if [ -e .fetched_source ]; then
rm .fetched_source || perror "Could not delete ${TARGET_DIR}/kernel/.fetched_source"
fi
if [ -e .built ]; then
rm .built || perror "Could not delete ${TARGET_DIR}/kernel/.built"
fi
if [ -e ksrc ]; then
unlink ksrc || perror "Could not unlink ${TARGET_DIR}/kernel/ksrc."
fi
else
pinfo "Not confirmed, not cleaning kernel module."
fi
pinfo "Done cleaning kernel."
}
