#!/bin/bash
# -----------------------------------------------------------------------------
#
# Copyright (c) 2011 - OpenSLX GmbH
#
# This program is free software distributed under the GPL version 2.
# See http://openslx.org/COPYING
#
# If you have any feedback please consult http://openslx.org/feedback and
# send your suggestions, praise, or complaints to feedback@openslx.org
#
# General information about OpenSLX can be found at http://openslx.org/
# -----------------------------------------------------------------------------
#
# This file contains the functions needed to setup a target.
# The main function is 'generate_target' which will go over all
# activated modules (symlinks found under remote/targets/<target>)
# and run the module-specific 'fetch_source', 'build' and 'post_copy'
# functions of the modules.
#
# The functions defined here are module-independent, meaning they
# can process any module, as long as following requirements are met:
#
# - Modules need to have a build-script and a config file,
# i.e. remote/module/kernel/kernel.build
# and remote/module/kernel/kernel.conf
# as these will be read by 'process_module'.
#
# - Modules need to generate a 'build'-directory in their
# own sub-directory (i.e. remote/modules/sshd/build/
# as 'copy_files_with_deps' will look for the files in
# that directory ONLY.
#
# - Several information have to be set in the config file of
# a module:
# * REQUIRED_BINARIES - list of binaries
# * REQUIRED_FILES - list of regular files
# * REQUIRED_DIRECTORIES - list of directories
# * REQUIRED_SYSTEM_FILES - files to be copied directly from the system
# * REQUIRED_INSTALLED_PACKAGES - list of packages needed at build-time
# * REQUIRED_CONTENT_PACKAGES - list of packages where the contents are to be copied
# * (optional variables needed by the module-build-script)
#
# NOTE: REQUIRED_BINARIES, REQUIRED_FILES, REQUIRED_FILES will be ONLY looked
# for in the module's own build directory!
#
# Optionally, a module can have a static data directory under
# remote/modules/<module>/data which will be copied as is to the target build directory.
#
# Moreover modules do not need to specify the dynamic dependencies of given binaries
# as these will be determined automaticly by the helper function 'get_dynamic_dependencies'
# -----------------------------------------------------------------------------
#
MODE_DIR="${ROOT_DIR}/remote"
MODULES_DIR="${MODE_DIR}/modules"
EXPORT_DIR="/export/build"
# Keep track of processed modules
PROCESSED_MODULES=""
initial_checks () {
detect_distribution
}
#
# generic function to read the config file of the current $MODULE
#
read_config () {
# unset previous variables from other config files
for VARNAME in ${!REQUIRED_*}; do
unset $VARNAME
done
local MODULE_CONFIG="${MODULE_DIR}/${MODULE}.conf"
# sanity checks
[ ! -e "${MODULE_CONFIG}" ] && perror "Config for '$MODULE' not found."
validate_config "$MODULE_CONFIG"
# source the config file
. "${MODULE_CONFIG}" || perror "Sourcing '${MODULE_CONFIG}' failed."
# sanity checks - no distribution specific things in the global config
local CHECK
for CHECK in REQUIRED_CONTENT_PACKAGES REQUIRED_INSTALLED_PACKAGES; do
[ -n "${!CHECK}" ] && perror "$CHECK is set in ${MODULE}.conf, but distribution specific config has to go to one of these files:\n$PRINT_SYS_VERSIONS"
done
local FILE
for FILE in $SYS_VERSIONS; do
if [ -e "${MODULE_CONFIG}.${FILE}" ]; then
# a specific tool.conf seems to exist, use it to override/extend certain vars
validate_config "${MODULE_CONFIG}.${FILE}"
. "${MODULE_CONFIG}.${FILE}" || perror "Sourcing '${MODULE_CONFIG}.${FILE}' failed."
pinfo "Sourced distribution specific ${MODULE_CONFIG}.${FILE}"
break
fi
done
}
validate_config () {
local INVALID=$(grep -h -E -o '^\s*\w+=' "$1" | grep -v -E '^\s*REQUIRED_')
[ -z "$INVALID" ] && return 0
local i
for i in $INVALID; do
pwarning "Invalid variable $i"
done
perror "$1 contains invalid variables. All variables must start with REQUIRED_"
}
#
# generic function to read the build script of the current $MODULE
#
read_build () {
local BUILD_SCRIPT="${MODULE_DIR}/${MODULE}.build"
[ ! -e "${BUILD_SCRIPT}" ] && perror "Build script for specified tool not found."
. "${BUILD_SCRIPT}" || perror "Sourcing '${BUILD_SCRIPT}' failed."
}
export_builds() {
[ ! -d "${MODE_DIR}/builds" ] && perror "No ${MODE_DIR}/builds, nothing to export."
pinfo "Mounting ${MODE_DIR}/builds to ${EXPORT_DIR}. This will make the local builds syncable from another machine."
[ ! -d "${EXPORT_DIR}" ] && mkdir -p "${EXPORT_DIR}"
mount --bind ${MODE_DIR}/builds ${EXPORT_DIR} || perror "Failed to bind mount ${MODE_DIR}/builds to ${EXPORT_DIR}"
}
#
# main function which copies all files, directories, binaries and external libraries to TARGET_BUILD_DIR
#
copy_files_with_deps () {
[ ! -d "$MODULE_BUILD_DIR" ] && pinfo "No build directory found, skipping dependency copying" && return 0
cd "$MODULE_BUILD_DIR"
COPYFILES_LIST="list_wanted_stage3.2"
[ -e "${COPYFILES_LIST}" ] && rm "${COPYFILES_LIST}"
[ ! -z "${REQUIRED_BINARIES}" ] && pinfo "Gathering required binaries from config file..."
local OPTIONAL
for FILENAME in ${REQUIRED_BINARIES}
do
if [[ "$FILENAME" == @* ]]; then
OPTIONAL="yes"
FILENAME=$(echo "$FILENAME" | cut -c 2- )
else
OPTIONAL="no"
fi
local FILE_CANDIDATES=$( find . -name "${FILENAME}" -a \( -type f -o -type l \) )
# only do if more than one candidate found
if [ "$(echo $FILE_CANDIDATES | wc -w)" -gt 1 ]; then
pdebug "Candidates for $FILENAME are: $(echo $FILE_CANDIDATES)"
local FINAL_LIST=""
for FILE in $FILE_CANDIDATES; do
local TESTFILE="$(readlink -f "$FILE")"
pdebug " $FILE leads to $TESTFILE"
[ -f "$TESTFILE" -a -x "$TESTFILE" ] && [ "x$(grep -l -E '^(.ELF|#!)' "$TESTFILE")" != "x" ] && FINAL_LIST="$FINAL_LIST $FILE"
done
FINAL_LIST=$(trim "$FINAL_LIST")
if [ -z "$FINAL_LIST" ]; then
pwarning "\tNo Binary found for ${FILENAME}. Skipping."
continue
fi
if [[ "$FINAL_LIST" == *" "* ]]; then
pwarning "Found more than one match for required file '$FILENAME': $FINAL_LIST"
else
pdebug "\tFound ${FILENAME} at ${FILE}"
fi
elif [ "$OPTIONAL" == "no" -a "$(echo $FILE_CANDIDATES | wc -w)" -eq 0 ]; then
perror "Could not find required binary $FILENAME"
else
# one candidate
FINAL_LIST=${FILE_CANDIDATES}
fi
for FILE in $FINAL_LIST; do
pdebug "* $FILE"
strip "$FILE" || pwarning "Could not strip '${FILE}'"
get_link_chain "${MODULE_BUILD_DIR}/${FILE}" "${MODULE_BUILD_DIR}" >> "${COPYFILES_LIST}"
get_dynamic_dependencies -l "${MODULE_BUILD_DIR}" "${FILE}" >> "${COPYFILES_LIST}"
done
done
[ ! -z "${REQUIRED_LIBRARIES}" ] && pinfo "Gathering required libraries from config file..."
for LIB in ${REQUIRED_LIBRARIES}
do
for LOCATION in $(find . -name ${LIB}.so\*)
do
pdebug "* $LOCATION"
strip "$LOCATION" || pwarning "Could not strip '${LOCATION}'"
get_link_chain "${MODULE_BUILD_DIR}/${LOCATION}" "${MODULE_BUILD_DIR}" >> "${COPYFILES_LIST}"
get_dynamic_dependencies -l "${MODULE_BUILD_DIR}" "${LOCATION}" >> "${COPYFILES_LIST}"
done
done
[ ! -z "${REQUIRED_DIRECTORIES}" ] && pinfo "Gathering required directories from config file..."
local CURRENT_PWD=$(pwd) # Prevent calling pwd 50000 times inside the loop below
for ENTRY in ${REQUIRED_DIRECTORIES}
do
pdebug "* .$ENTRY"
echo "./${ENTRY}" >> "${COPYFILES_LIST}"
for BIN in $(find ".${ENTRY}" -type f -not -name '*.a' | xargs grep -l '^.ELF')
do
#pdebug "\tSearching libs for ${BIN}..."
get_link_chain "${MODULE_BUILD_DIR}/${BIN}" "${MODULE_BUILD_DIR}" >> "${COPYFILES_LIST}"
get_dynamic_dependencies -l "${MODULE_BUILD_DIR}" "${BIN}" >> "${COPYFILES_LIST}"
done
done
[ ! -z "${REQUIRED_FILES}" ] && pinfo "Gathering required files from config file..."
for ENTRY in ${REQUIRED_FILES}
do
get_link_chain "${MODULE_BUILD_DIR}/${ENTRY}" "${MODULE_BUILD_DIR}" >> "${COPYFILES_LIST}"
get_dynamic_dependencies -l "${MODULE_BUILD_DIR}" "${MODULE_BUILD_DIR}/.${ENTRY}" >> "${COPYFILES_LIST}"
done
#copy to initramfsdir
pdebug "File list generated at ${MODULE_BUILD_DIR}/${COPYFILES_LIST}."
if [ -s "$COPYFILES_LIST" ]; then
local CLISTCOUNT=$(cat "$COPYFILES_LIST" | wc -l)
pinfo "Copying $CLISTCOUNT files to '${TARGET_BUILD_DIR}'."
tarcopy "$(cat "$COPYFILES_LIST"|sort -u)" "${TARGET_BUILD_DIR}"
fi
}
#
#
# main public function. Requires the TARGET to be given as argument.
# this will simply go over all the modules as found in the remote/target/<TARGET>
# and run following functions:
#
#
generate_target() {
initial_checks
TARGET=$1 && shift
TARGET_DIR="${MODE_DIR}/targets/${TARGET}"
TARGET_BUILD_DIR="${MODE_DIR}/builds/${TARGET}"
[ -e "${ROOT_DIR}/logs/${TARGET}.size" ] && . "${ROOT_DIR}/logs/${TARGET}.size" || echo "declare -A BUILD_SIZE" >> "${ROOT_DIR}/logs/${TARGET}.size"
[ -d "$TARGET_DIR" ] || perror "Given target directory does not exist: $TARGET_DIR"
[[ "$TARGET" == "builds" || "$TARGET" == "modules" ]] && \
perror "Target directory cannot be named 'builds' or 'modules'."
pinfo "Generating '$TARGET_BUILD_DIR' for '$TARGET'"
# if no arguments assume all.
if [ "x$1" = "x" -o "x$1" = "xall" ]; then
MODULES=$(ls ${TARGET_DIR})
set -- $MODULES
else
# tools = arguments given
MODULES=$@
fi
pinfo "Activated modules in '${TARGET}':"
pinfo "\t$(echo ${MODULES})"
# we need a bit of special logic for kernel specifically
# to check if the built version is corresponding to the
# current kernel version. If not we had a kernel update
# and need to ask the user what to do
if [[ "$MODULES" == *kernel* ]]; then
# if so, check what kernels have been built
if [ -e "${MODULES_DIR}/kernel/ksrc/KVERSION" ]; then
local BUILT_KERNEL_VERSION=$(cat "${MODULES_DIR}/kernel/ksrc/KVERSION")
if [[ "${BUILT_KERNEL_VERSION}" != "${KERNEL_CURRENT_VERSION}" ]]; then
pinfo "The built kernel (${BUILT_KERNEL_VERSION}) is older than the running one (${KERNEL_CURRENT_VERSION})!"
# cleaning the module is enough as it forces to rebuild.
[[ "${AUTO_UPDATE_KERNEL}" == 1 ]] && pinfo "Auto-Udpate Kernel..." && clean_kernel_module --force
fi
elif [ -e "${MODULES_DIR}/kernel/ksrc" ]; then
# check if ksrc is there, since that would be problematic
pwarning "${MODULES_DIR}/kernel/ksrc exists, but KVERSION is not found!"
pwarning "Kernel was probably build with an older version of mltk."
pwarning "If errors arise, either write your KVERSION manually or clean the kernel."
fi
fi
# copy basic libs
pinfo "Copying libc and ld-linux used by ${SHELL}"
tarcopy "$(list_basic_libs)" "${TARGET_BUILD_DIR}"
[ -d "$TARGET_BUILD_DIR" ] && TARGET_BUILD_SIZE=$(du -bc "${TARGET_BUILD_DIR}" | awk 'END {print $1}') || TARGET_BUILD_SIZE=0
# now iterate over given tools and copy them
while (( "$#" )); do
process_module "$1"
shift
done
post_process_target
pinfo "Target completed. Total size: $(du -bsh "${TARGET_BUILD_DIR}" | awk 'END {print $1}')"
TOOL_STR=""
}
process_module() {
[ "$#" -ne "1" ] && perror "process_module: want 1 param."
local MODULE="$1"
[[ "$PROCESSED_MODULES" == *"!${MODULE}!"* ]] && return # Already processed this module
PROCESSED_MODULES="${PROCESSED_MODULES}!${MODULE}!"
local MODULE_DIR="${TARGET_DIR}/${MODULE}"
local MODULE_BUILD_DIR="${MODULE_DIR}/build"
local TOOL_STR=""
pinfo ">>>>>>>>>>>>>>>>> Processing module [ $MODULE ]"
TOOL_STR="[${MODULE}]"
if [ -d "${MODULE_DIR}" ]; then
cd "${MODULE_DIR}" || perror "Module dir '${MODULE_DIR}' seems to exist, but cd to it failed."
pinfo "## Reading config"
read_config
# Check if this module has a dependency that wasn't built yet:
if [ ! -z "$REQUIRED_MODULES" ]; then
pinfo "$MODULE depends on ${REQUIRED_MODULES}...."
for DEP in $REQUIRED_MODULES; do
process_module "$DEP"
done
# Read old config again, as it got overwritten by the deps
cd "${MODULE_DIR}" || perror "Tool dir '${MODULE_DIR}' seems to exist, but cd to it failed (after building deps)."
read_config
pinfo "<<<<<<<<<<<<<<<<< Dependency modules processed, back to module [ $MODULE ]"
fi
pinfo "## Reading build"
read_build
pinfo "## Installing dependencies"
install_dependencies
pinfo "## Fetching source"
[ -e "${MODULE_DIR}/.fetched_source" ] || { fetch_source && touch "${MODULE_DIR}/.fetched_source"; }
if [ -e "${MODULE_DIR}/.built" -a -d "${MODULE_BUILD_DIR}" ]; then
pinfo "# built-flag is set, skipping build..."
else
pinfo "## Building"
mkdir -p "${MODULE_BUILD_DIR}" || perror "Could not create build dir"
build # calls perror if something fails, no need to do that here
touch "${MODULE_DIR}/.built" || pwarning "Error setting built-flag"
fi
# remove *.la files as they might confuse libtool/linker of other tool packages
[ -d "${MODULE_BUILD_DIR}" ] && find "${MODULE_BUILD_DIR}" -name '*.la' -exec rm -f {} \;
[ -d "${TARGET_BUILD_DIR}" ] && TARGET_BUILD_SIZE=$(du -bc "${TARGET_BUILD_DIR}" | awk 'END {print $1}') || TARGET_BUILD_SIZE=0
pinfo "## Copying files with dependencies"
copy_files_with_deps
pinfo "## Copying required system files"
copy_system_files
pinfo "## Copying static module files"
copy_static_data
pinfo "## Post copy"
post_copy
# Sanity checks
[ -e "$TARGET_BUILD_DIR/var/run" -a ! -L "$TARGET_BUILD_DIR/var/run" ] && perror "Messup datected: $TARGET_BUILD_DIR/var/run exists and is not a symlink!"
[ -e "$TARGET_BUILD_DIR/var/lock" -a ! -L "$TARGET_BUILD_DIR/var/lock" ] && perror "Messup datected: $TARGET_BUILD_DIR/var/lock exists and is not a symlink!"
[ -n "$(ls -A "$TARGET_BUILD_DIR/run")" ] && perror "Messup detected: $TARGET_BUILD_DIR/run is not empty. You cannot place static files there, use /etc/tmpfiles.d instead!"
# set MODULE_BUILD_SIZE
calc_size
pinfo "Module completed. Total size: ${MODULE_BUILD_SIZE}"
else
pwarning "Module directory for '$MODULE' not found."
fi
}
post_process_target() {
local TOOL_STR="$TOOL_STR post_process_target:"
if [ -d "${TARGET_BUILD_DIR}/lib/systemd" ]; then
pwarning "/lib/systemd exists in the generated target. It is not advisable to fetch existing"
pwarning "systemd service files from the template system, as it might conflict with generated"
pwarning "ones. In fact, some systems don't even use systemd so there might be no existing"
pwarning "service files. Please generate your own service file and place it in /etc/systemd."
pwarning "All files in /lib/systemd will be ignored."
fi
# figure out all relevant ld-paths
pinfo "Running ldconfig"
local LDTEMP="$TARGET_BUILD_DIR/ldconf-temp"
grep -h '^\s*/' /etc/ld.so.conf /etc/ld.so.conf.d/* > "$LDTEMP"
# TODO: Echo other stuff to $LDTEMP (but figure out if first or last match wins)
ldconfig -v -r "${TARGET_BUILD_DIR}" -f "/ldconf-temp"
unlink "$LDTEMP"
}
clean_modules() {
TARGET=$1
shift
TARGET_DIR="${MODE_DIR}/targets/${TARGET}"
TARGET_BUILD_DIR="${MODE_DIR}/builds/${TARGET}"
[ -d $TARGET_DIR ] || perror "Given target directory does not exist: $TARGET_DIR"
if [ "x$1" = "x" -o "x$1" = "xall" ]; then
if [ -e ${ROOT_DIR}/logs/${TARGET}.size ]; then
rm "${ROOT_DIR}/logs/${TARGET}.size" || perror "Could not delete logs/${TARGET}.size"
fi
if [ -d ${TARGET_BUILD_DIR} ]; then
pinfo "Cleaning '${TARGET_BUILD_DIR}'" \
&& rm -rf "${TARGET_BUILD_DIR}"/* \
|| perror "Error deleting $TARGET_BUILD_DIR"
fi
set -- $(ls ${TARGET_DIR} | grep -v kernel)
fi
cd "$TARGET_DIR"
while (( "$#" )); do
clean_module $(readlink -f $1)
shift
done
cd - &> /dev/null
# libck-connector cleanup
rm -f "$TARGET_DIR/post-patch/ck-connector/libck-connector.so"
}
clean_module() {
# if kernel is to be cleaned, do it separately and return
[ "x$(basename $1)" == "xkernel" ] && clean_kernel_module && return
pinfo "Cleaning '$1'..."
local MODULE_DIR=$1
if [ -e ${MODULE_DIR}/.built ]; then
rm "${MODULE_DIR}/.built" || perror "Could not clear built flag"
fi
if [ -e ${MODULE_DIR}/.fetched_source ]; then
rm "${MODULE_DIR}/.fetched_source" || perror "Could not clear fetched_source flag"
fi
if [ -d ${MODULE_DIR}/build ]; then
rm -rf "${MODULE_DIR}/build" || perror "Could not delete build path"
fi
if [ -d ${MODULE_DIR}/src ]; then
rm -rf "${MODULE_DIR}/src" || perror "Could not delete src path"
fi
if [ -e ${MODULE_DIR}/list_dpkg_output ]; then
rm "${MODULE_DIR}/list_dpkg_output" || perror "Could not delete list_dpkg_output"
fi
if [ -e ${MODULE_DIR}/list_binaries_and_files ]; then
rm "${MODULE_DIR}/list_binaries_and_files" || perror "Could not delete list_binaries_and_files"
fi
}
clean_kernel_module() {
local CLEAN=0
if [ "x$1" == "x--force" ]; then
CLEAN=1
else
pinfo "It will take a long time to rebuild, are you sure? [y/N]"
read USER_INPUT
[ "x$USER_INPUT" == "xy" ] && CLEAN=1
fi
pinfo "Cleaning kernel module (including sources and compiled stuff)."
if [[ $CLEAN == 1 ]]; then
cd "${TARGET_DIR}/kernel" || perror "Could not cd to ${TARGET_DIR}/kernel"
if [ -e build ]; then
rm -rf build || perror "Could not delete ${TARGET_DIR}/kernel/build"
fi
# little workaround to actually list names of entries
# (if only a dir exists, ls kernel/linux-* would just list
# the contents of that directory....)
for FILE in $(ls | grep linux); do
rm -rf $FILE || perror "Could not delete $TARGET_DIR}/$FILE"
done
# finally clean the config generated
if [ -e openslx.config ]; then
rm openslx.config || perror "Could not delete ${TARGET_DIR}/kernel/openslx.config"
fi
if [ -e .fetched_source ]; then
rm .fetched_source || perror "Could not delete ${TARGET_DIR}/kernel/.fetched_source"
fi
if [ -e .built ]; then
rm .built || perror "Could not delete ${TARGET_DIR}/kernel/.built"
fi
if [ -L ksrc ]; then
unlink ksrc || perror "Could not unlink ${TARGET_DIR}/kernel/ksrc."
fi
else
pinfo "Not confirmed, not cleaning kernel module."
fi
pinfo "Done cleaning kernel."
}