path: root/satellit_upgrader/updater.template.sh

#!/bin/bash

# Use special param to make sure we're running under bash (in case user does "sh install.sh")
[ "x$1" != "x--exec-self" ] && exec /bin/bash "$0" --exec-self "$@"

if [ ! -d "/opt/dmsd" ] && [ ! -d "/srv/openslx/www/slx-admin" ]; then
	echo "This system doesn't look like a bwLehrpool satellite server"
	echo "(Or this updater is really old)"
	exit 1
fi

if [ "$(lsb_release -si)" != "Debian" ]; then
	echo "This updater only supports Debian-based satellite servers"
	exit 1
fi

IGNORE_ERRORS=
DO_UPDATE=
FORCE=
while [ $# -gt 0 ]; do
	[ "$1" = "--ignore-errors" ] && IGNORE_ERRORS=jup
	[ "$1" = "--update" ] && DO_UPDATE=pwarning
	[ "$1" = "--force" ] && FORCE=true
	shift
done
readonly IGNORE_ERRORS

CODENAME="$( lsb_release -sc )"
MAJOR="$( lsb_release -sr )"
MAJOR="${MAJOR%%.*}"
if ! (( MAJOR > 0 )); then
	case "$CODENAME" in
		bullseye) MAJOR=11 ;;
		bookworm) MAJOR=12 ;;
		trixie)   MAJOR=13 ;;
	esac
fi
if ! (( MAJOR > 0 )); then
	echo "Could not determine Debian version"
	exit 1
fi

delay=
readonly MAJOR CODENAME

if [[ "$(uname -m)" == i?86 ]]; then
	echo "Please note: This bwLehrpool server is still running on a 32 Bit OS install."
	echo "This release will be the last one with 32 Bit support. Please consider backing"
	echo "up your configuration and setting up a fresh server with the latest"
	echo "bwLehrpool server image."
	echo ""
	delay=1
fi

if (( MAJOR < 9 )); then
	echo "This Version of the bwLehrpool satellite server is too old!"
	echo "Back up your existing configuration via slx-admin and import it into a"
	echo "fresh install, or manually dist-upgrade this server."
	exit 1
fi
if (( MAJOR > 12 )); then
	echo "This Version of the bwLehrpool satellite server is too new!"
	exit 1
fi
if (( MAJOR == 9 )); then
	echo "This server is still running Debian ${MAJOR}. Debian ${MAJOR} does not receive any"
	echo "security patches since 2022-06-20. It is strongly recommended to upgrade"
	echo "to at least Debian 10 (ideally, 12) before installing this bwLehrpool server update,"
	echo "either by doing a dist-upgrade, or by backing up the configuration and"
	echo "importing it into a freshly installed server."
	if [ -z "$FORCE" ]; then
		echo ""
		echo "If you still want to apply this update for now, add the '--force' command"
		echo "line option, i.e. $0 --update --force"
		exit 1
	fi
	delay=1
fi

if [ -n "$delay" ]; then
	echo -n "About to begin update in 10 seconds (press Ctrl-C to abort).."
	for _ in {1..10}; do
		sleep 1
		echo -n .
	done
	echo .
fi

unset TMPDIR
[ -n "$TMPDIR" ] && exit 1

declare -rg SELFPID=$$
perror () {
	if [ -n "$IGNORE_ERRORS" ]; then
		echo -n -e '\033[01;31m[ERROR]\033[00m '
		echo "$@"
		return 0
	fi
	echo -n -e '\033[01;31m[FATAL]\033[00m '
	echo "$@"
	[ -n "$TMPDIR" ] && [ -d "$TMPDIR" ] && rm -rf -- "$TMPDIR"
	[ "$$" != "$SELFPID" ] && kill "$SELFPID"
	exit 1
}

pwarning () {
	echo -n -e '\033[01;33m[WARNING]\033[00m '
	echo "$@"
}

if [ "$UID" != "0" ]; then
	perror "Must be running as root"
fi

# ** Extract value from text file containing key=value pairs
extractfield () {
	grep -m1 "^\s*$2\b" "$1" | awk -F '=' '{print $2}' | sed 's/\s//g'
}

# ** Wrap mysql command line client so we're always using the deb-sys-maint credentials
mysql () {
	"$(which mysql)" --defaults-extra-file=/etc/mysql/debian.cnf --default-character-set=utf8mb4 "$@"
}

generate_password() {
	tr -dc _A-Za-z0-9 < /dev/urandom | head -c 16
}

# ** Restart given systemd service, warn if it fails but do not bail out
restart_service () {
	if ! systemctl restart "$1"; then
		pwarning "Could not restart service $1 - !! YOU SHOULD REBOOT THE SERVER !!"
		systemctl status "$1"
	fi
}

# ** Recursively set permissions of given directory to 0755/0644 optionally setting owner/group
fixperms () {
	[ -e "$1" ] || perror "normalize_perms: $1 does not exist"
	chmod -R a-rw,u+rwX,go+rX "$1"
	if [ -n "$2" ]; then
		chown -R "$2" "$1" || pwarning "Could not change owner of $1 to $2"
	fi
}

# diffcp <module> <file> [message]
# diffcp "dnbd3" "/opt/openslx/foo.txt" ["Installing new foo file"]
# return 0 (true) if the file was different and thus copied
diffcp () {
	local SRC="$FILEDIR/$1/$2"
	local DST="/$2"
	[ -e "$SRC" ] || perror "$1/$2 not included in updater"
	[ -f "$SRC" ] || perror "$1/$2 included in updater, but is not regular file."
	if [ -e "$DST" ]; then
		[ -f "$DST" ] || perror "$DST exists and is not a regular file, cannot replace."
		cmp -s "$SRC" "$DST" &>/dev/null && return 1 # cmp returns 0 if files are equal
	fi
	if [ -n "$3" ]; then
		echo "$3"
	else
		echo "Installing $2"
	fi
	mkdir -p "$( dirname "$DST" )"
	cp -d "$SRC" "$DST" 2> /dev/null || cp -d -f "$SRC" "$DST" || perror "Could not copy '$1/$2' to '$DST'"
	return 0
}

# install_files <module>
# returns 0 (true) if at least one file was copied
install_files () {
	local SRC="$FILEDIR/$1"
	local files file ret
	[ -d "$SRC" ] || perror "NO SUCH DIRECTORY: $SRC"
	readarray -t -d '' files < <( find "$SRC" \( -type f -o -type l \) -print0 )
	ret=1
	for file in "${files[@]}"; do
		diffcp "$1" "${file#$SRC}" && ret=0
	done
	return $ret
}

# failprint apt-get install foobar
# Only prints command output to stdout if exit code is not 0
failprint () {
	local tf ret
	tf=$(mktemp)
	"$@" &> "$tf"
	ret=$?
	[ "$ret" != 0 ] && cat "$tf"
	rm -f -- "$tf"
	return "$ret"
}

aptupdate=false
aptinst () {
	local pkg=$1
	if ! dpkg -l "$pkg" 2> /dev/null | grep -q "^ii\\s.*$pkg"; then
		if ! $aptupdate; then
			echo "Updating apt db"
			# Kill this file; was accidentally put there on stretch; on jessie it's not
			# supported anymore as of 2019-03-20
			rm -f -- "/etc/apt/sources.list.d/10-added_repos.list"
			# No more updates for pre-point release updates on jessie
			# But re-enable (in case it was dist-upgraded)
			sed -r -i '/^\s*#.*jessie-updates/s/^\s*#//;s/jessie-updates/'"$CODENAME"'-updates/' "/etc/apt/sources.list"
			failprint apt-get update || pwarning "Cannot run apt-get update"
			aptupdate=true
		fi
		echo "Installing $pkg"
		if ! failprint apt-get install -y "$pkg"; then
			pwarning "*"
			pwarning "* Warning: Could not install $pkg!"
			pwarning "*"
			pwarning "* Please do either \"apt-get install $pkg\" by hand or"
			pwarning "* install $pkg otherwise."
			pwarning "*"
			return 1
		fi
	else
		failprint apt-mark manual "$pkg"
	fi
	return 0
}

# ** Constants - to be patched by the packaging script
declare -rg TARGET_WEBIF_VERSION="%TARGET_WEBIF_VERSION%"
declare -rg TGZ_SLXADMIN="%TGZ_SLXADMIN%"
declare -rg TGZ_DOZMOD="%TGZ_DOZMOD%"
declare -rg TGZ_TASKMANAGER="%TGZ_TASKMANAGER%"
declare -rg TGZ_TFTP="%TGZ_TFTP%"
declare -rg TGZ_IPXE="%TGZ_IPXE%"
declare -rg TGZ_LDADP="%TGZ_LDADP%"
declare -rg TGZ_DNBD3="%TGZ_DNBD3%"
declare -rg PAYLOAD_OFFSET="%PAYLOAD_OFFSET%"
declare -rg SLXADMIN_FOOTER="%SLXADMIN_FOOTER%"

# ** Constants - hardcoded or determined at runtime
declare -rg PATH_SLXADMIN="/srv/openslx/www/slx-admin"
declare -rg PATH_SLXADMINCACHE="/var/cache/slx-admin"
declare -rg PATH_DOZMOD="/opt/dmsd"
declare -rg PATH_TASKMANAGER="/opt/taskmanager"
declare -rg PATH_TFTP="/srv/openslx/tftp"
declare -rg PATH_IPXE="/opt/openslx/ipxe"
declare -rg PATH_LDADP="/opt/ldadp"
declare -rg PATH_DNBD3="/opt/openslx/dnbd3"
declare -rg PATH_SLXSTORE="/srv/openslx/nfs"

# ** Check if constants have been filled, bail out otherwise
if [ -z "$TARGET_WEBIF_VERSION" ] || [[ "$TARGET_WEBIF_VERSION" == %*% ]]; then
	[ -n "$TGZ_SLXADMIN" ] && perror "Bad upgrader: TARGET_WEBIF_VERSION not set"
fi
if [ -z "$PAYLOAD_OFFSET" ] || [[ "$PAYLOAD_OFFSET" == %*% ]]; then
	perror "Bad upgrader: PAYLOAD_OFFSET not set"
fi

# **********************************************************
export DEBIAN_FRONTEND=noninteractive
declare -A restart
daemon_reload=

# Get current webif version
if [ -n "$TGZ_SLXADMIN" ]; then
	declare -rg CURRENT_WEBIF_VERSION="$( < "${PATH_SLXADMIN}/version"  tail -n 1 )"
	# Print state of affairs
	echo "bwLehrpool satellite server update"
	echo -n "Currently installed version: "
	sed -r 's/([0-9]{4})([0-9]{3})([0-9]{2})/\1-\2-\3/' <<<"$CURRENT_WEBIF_VERSION"
	echo -n "This update package:         "
	sed -r 's/([0-9]{4})([0-9]{3})([0-9]{2})/\1-\2-\3/' <<<"$TARGET_WEBIF_VERSION"
	# Now if this appears to be a downgrade wait 5 seconds...
	if [ -z "$CURRENT_WEBIF_VERSION" ] || [ "$TARGET_WEBIF_VERSION" = "missing" ] || [ "$CURRENT_WEBIF_VERSION" -le "$TARGET_WEBIF_VERSION" ]; then
		: # Negate so NaN triggers else branch
	else
		pwarning "This update seems to be older than the version you're currently running."
		[ -z "${FORCE}" ] && perror "Pass --force to install anyways."
		echo -n "Press Ctrl+C to cancel update..."
		for i in 5 4 3 2 1; do
			sleep 1
			echo -n "${i}.."
		done
	fi
	# Make sure --update was passed
	if [ -z "$DO_UPDATE" ]; then
		echo "If you want to install this update, run again with --update parameter:"
		echo "eg. sh $0 --update"
		exit 0
	fi
fi

# ************** Extract payload ***************************
declare -rg TMPDIR=$(mktemp -d)
export TMPDIR
[ -z "$TMPDIR" ] && perror "Could not create temporary directory for installer"
dd "bs=$PAYLOAD_OFFSET" "if=$0" skip=1 | tar -z -x -C "$TMPDIR"
RET=$?
[ "$RET" -ne 0 ] && perror "Extracting installer payload failed with exit code $RET"
declare -rg FILEDIR="$TMPDIR/files"
chmod 0700 "$FILEDIR" # In case we patch passwords into files there

# **********************************************************
# **********************************************************
# **********************************************************

# **** Start generating dhparam early as it takes a long time **********************
declare -rg DHPARAM="/etc/lighttpd/dhparam.pem"
DH_PID=
if ! [ -s "$DHPARAM" ]; then
	openssl dhparam -out "$DHPARAM" 2048 >/dev/null 2>&1 &
	DH_PID=$!
fi

# **************** Fix tmpdelete cronjob *******************
if [ -f "/etc/cron.daily/tmpdelete.sh" ]; then
	[ -f "/etc/cron.daily/tmpdelete" ] || mv "/etc/cron.daily/tmpdelete.sh" "/etc/cron.daily/tmpdelete"
	rm -f -- "/etc/cron.daily/tmpdelete.sh"
fi

if (( MAJOR == 9 )); then
	java="7" # highest UNWANTED version
	phpmajor="7" # WANTED
	php="-1" # highest UNWANTED version minor (7.x)
	mysql="10.1" # WANTED version
elif (( MAJOR == 10 )); then
	java="10" # highest UNWANTED version
	phpmajor="7" # WANTED
	php="2" # highest UNWANTED version minor (7.x)
	mysql="10.3" # WANTED version
elif (( MAJOR == 11 )); then
	java="10" # highest UNWANTED version
	phpmajor="7" # WANTED
	php="3" # highest UNWANTED version minor (7.x)
	mysql="10.5" # WANTED version
elif (( MAJOR == 12 )); then
	java="16" # highest UNWANTED version
	phpmajor="8" # WANTED
	php="1" # highest UNWANTED version minor (8.x)
	mysql="10.11" # WANTED version
else
	perror "Abort"
fi

echo "Cleaning up old packages..."
failprint apt autoremove -y

# Make sure current java is installed
for ver in $(seq 6 "$java"); do
	dpkg -l "openjdk-${ver}-*" 2> /dev/null | grep -q '^ii' && apt remove -y "openjdk-${ver}-*"
done
aptinst default-jre-headless

# Same for php
dpkg -l 'php5-*' | grep -q '^ii' && apt remove -y "php5-*"
for ver in $( seq 7 "$(( phpmajor - 1 ))" ); do
	dpkg -l "php${ver}.?-*" | grep -q '^ii' && apt remove -y "php${ver}.?-*"
done
for ver in $( seq 0 "$php" ); do
	dpkg -l "php${phpmajor}.${ver}-*" 2> /dev/null | grep -q '^ii' && apt remove -y "php${phpmajor}.${ver}-*"
done
aptinst php-cgi
aptinst php-cli
aptinst php-curl
aptinst php-json
aptinst php-mysql
aptinst php-mbstring
aptinst php-xml
aptinst php-soap

# MySQL...
for ver in $( dpkg-query -W -f='${Package}\n' "mysql-server-*" "mariadb-server-*" ); do
	[[ "$ver" == *"-$mysql" ]] && continue
	[[ "$ver" =~ -[.0-9]+$ ]] || continue
	dpkg -l "$ver" 2> /dev/null | grep -q '^ii' && apt remove -y "$ver"
done
aptinst default-mysql-server

failprint apt autoremove -y

## Enable unix socket passwordless local connection (commented out since default in Debian 9+ anyways)
#mysql -e "UPDATE user SET plugin = 'unix_socket' WHERE Host = 'localhost' AND User = 'root'"

install_files "system-updates"

# ********************** Taskmanager ***********************
# Do this first in case one of the later updates depends on the taskmanager
# Config files
tm_restart=
tm_password=
if [ -s "$PATH_TASKMANAGER/config/config" ]; then
	tm_password="$(awk '$0 ~ /^password\s*=/ {gsub(/^password\s*=\s*/, ""); gsub(/\s*$/, ""); print $0; exit}' "$PATH_TASKMANAGER/config/config")"
	[ "$tm_password" = "%TM_OPENSLX_PASS%" ] && tm_password=
fi
if [ -z "$tm_password" ] && [ -s "${PATH_SLXADMIN}/config.php" ]; then
	tm_password="$(grep -m1 '^define.*CONFIG_TM_PASSWORD.*;' "${PATH_SLXADMIN}/config.php" | awk -F "'" '{print $4}')"
	[ "$tm_password" = "%TM_OPENSLX_PASS%" ] && tm_password=
fi
if [ -z "$tm_password" ]; then
	tm_password="$(generate_password)"
fi
if [ -n "$tm_password" ]; then
	sed -i "s/%TM_OPENSLX_PASS%/${tm_password}/" "$FILEDIR/taskmanager/$PATH_TASKMANAGER/config/config"
fi
mkdir -m 0750 -p "$PATH_TASKMANAGER/config"
diffcp "taskmanager" "$PATH_TASKMANAGER/config/config" && tm_restart=1
diffcp "taskmanager" "$PATH_TASKMANAGER/config/environment" && tm_restart=4
diffcp "taskmanager" "/etc/systemd/system/taskmanager.service" && tm_restart=111
# java app
if [ -n "$TGZ_TASKMANAGER" ]; then
	[ -e "$TMPDIR/$TGZ_TASKMANAGER" ] || perror "$TGZ_TASKMANAGER missing from payload"
	echo "* Task manager"
	echo "Extracting new jar and data"
	# Replacement trick (see slxadmin)
	tar -x -C "$PATH_TASKMANAGER" -f "$TMPDIR/$TGZ_TASKMANAGER" || perror "Could not extract $TGZ_TASKMANAGER to $PATH_TASKMANAGER"
	rm -rf -- "$PATH_TASKMANAGER/data" "$PATH_TASKMANAGER/scripts" "$PATH_TASKMANAGER/plugins" "$PATH_TASKMANAGER/taskmanager.sh"
	tar -x -C "$PATH_TASKMANAGER" -f "$TMPDIR/$TGZ_TASKMANAGER"
	fixperms "$PATH_TASKMANAGER" root:root
	chmod -R a+x "$PATH_TASKMANAGER/scripts"
	# Where we place the downloadable iPXE images
	mkdir -p -m 0755 "/srv/openslx/www/boot/download"
	fixperms "/srv/openslx/www/boot/download" taskmanager:www-data
	tm_restart=1
	echo "Taskmanager upgrade complete"
fi
# Protect config with password in it
chmod 0640 "$PATH_TASKMANAGER/config/config"
chown -R root:taskmanager "$PATH_TASKMANAGER/config"
# sudo config
if diffcp "taskmanager" "/etc/sudoers.d/taskmanager" "Installing new sudo config for taskmanager"; then
	chmod 0440 "/etc/sudoers.d/taskmanager"
fi
if [ -n "$tm_restart" ]; then
	echo "Restarting Taskmanager..."
	systemctl daemon-reload
	restart_service taskmanager.service # Restart right away for above reason
fi

# ************************** IPXE **************************
echo "* iPXE"
IPXE_PID=
build=false
aptinst syslinux # Needed for genfsimg
if grep -qF 'https://git.openslx.org/openslx-ng/ipxe.git' "$PATH_IPXE/.git/config"; then
	echo "Updating existing clone"
	build=true
elif [ -n "$TGZ_IPXE" ]; then
	[ -e "$TMPDIR/$TGZ_IPXE" ] || perror "$TGZ_IPXE missing from payload"
	echo "Extracting new version from tarball"
	rm -rf -- "$PATH_IPXE"
	mkdir -p "$PATH_IPXE"
	tar -x -C "$PATH_IPXE" -f "$TMPDIR/$TGZ_IPXE" || perror "Could not extract $TGZ_IPXE to $PATH_IPXE"
	# Trigger default compile in the background so the next regen doesn't take too long
	build=true
fi
if $build; then
	# Bring up to date either way
	cd "$PATH_IPXE" || perror "ipxe directory not found ($PATH_IPXE)"
	fixperms "$PATH_IPXE" taskmanager:taskmanager
	# Reset modified config, older versions changed these
	echo "Running git..."
	sudo -n -u taskmanager git reset &> /dev/null
	sudo -n -u taskmanager git checkout src/config &> /dev/null
	sudo -n -u taskmanager git checkout openslx
	sudo -n -u taskmanager git pull --all &> /dev/null \
		|| (
			sudo -n -u taskmanager git reset --hard origin/openslx
			failprint sudo -n -u taskmanager git pull --all
			) \
		|| pwarning "Could not update iPXE -- let's hope the old version still works."
	echo "Adding bwlp config..."
	install_files "ipxe-bwlp"
	mkdir -p "/opt/openslx/ipxe/src/config/local/bwlp"
	cp /opt/openslx/ipxe-bwlp-config/*.h "/opt/openslx/ipxe/src/config/local/bwlp/"
	fixperms "$PATH_IPXE" taskmanager:taskmanager
	echo "Resetting pxe menu"
	mysql -e 'UPDATE openslx.property SET value = "invalid" WHERE name = "server-ip"' || pwarning "Could not reset pxe menu status; manual regeneration of menu required"
	{
		cd "$PATH_IPXE/src" && sudo -n -u taskmanager nice -n 10 make -j2 CONFIG=bwlp bin/undionly.kkkpxe > /dev/null 2>&1
	} &
	IPXE_PID=$!
fi
# Need mtools for creating floppy images for iPXE
aptinst mtools

# ************************** SLX-Admin *********************
if [ -n "$TGZ_SLXADMIN" ]; then
	[ -e "$TMPDIR/$TGZ_SLXADMIN" ] || perror "$TGZ_SLXADMIN missing from payload."
	echo "* SLX-Admin: $CURRENT_WEBIF_VERSION -> $TARGET_WEBIF_VERSION"
	# Test archive
	INCMODS=$(mktemp)
	tar tf "$TMPDIR/$TGZ_SLXADMIN"  | grep -oP '^modules-available/[^/]+/$' > "$INCMODS" || perror "Could not read $TGZ_SLXADMIN"
	# Remove old files - wipe selectively, just in case someone added a new module by hand
	rm -rf -- "$PATH_SLXADMIN/"{inc,apis,modules,templates,Mustache,script,style,modules-available/serversetup-bwlp,modules-available/support,modules-available/usermanagement,modules-available/baseconfig_partitions_cdn,modules-available/internetaccess}
	while read -r line || [[ -n "$line" ]]; do
		rm -rf -- "${PATH_SLXADMIN:?}/$line"
	done < "$INCMODS"
	tar -x -C "$PATH_SLXADMIN" -f "$TMPDIR/$TGZ_SLXADMIN" || perror "Could not extract $TGZ_SLXADMIN to $PATH_SLXADMIN"
	rm -f -- "$PATH_SLXADMIN/config.php.example"
	# New config.php supplied
	if [ -s "$FILEDIR/slxadmin/${PATH_SLXADMIN}/config.php" ]; then
		echo "Updating slx-admin config"
		# Get data from old config
		OLDFOOTER="$(grep -m 1 CONFIG_FOOTER "${PATH_SLXADMIN}/config.php"  | awk -F "'" '{print $4}' | sed 's/[ \[\]\(\)]*Update.*$//;s/#/+/g')"
		SQLPASS=$(grep -m1 '^define.*CONFIG_SQL_PASS.*;' "${PATH_SLXADMIN}/config.php" | awk -F "'" '{print $4}')
		# Patch config with new features using old mysql password
		if [ -z "$SQLPASS" ]; then
			pwarning "Could not extract old slx-admin mysql password - cannot update config :("
		else
			sed "s/%MYSQL_OPENSLX_PASS%/${SQLPASS}/g" "$FILEDIR/slxadmin/${PATH_SLXADMIN}/config.php" > "${PATH_SLXADMIN}/config.php"
		fi
		sed -i "s/%TM_OPENSLX_PASS%/${tm_password}/g" "${PATH_SLXADMIN}/config.php"
		# Update version info in footer
		[ -z "$OLDFOOTER" ] && OLDFOOTER="Unknown base version"
		FOOTER="$SLXADMIN_FOOTER"
		[[ "$FOOTER" == %*% ]] && FOOTER=
		if [ -z "$FOOTER" ]; then
			FOOTER="Build $TARGET_WEBIF_VERSION"
		fi
		FOOTER="$OLDFOOTER [Update: ${FOOTER}]"
		sed -i "s#^.*define.*CONFIG_FOOTER.*\$#define('CONFIG_FOOTER', '${FOOTER}');#" "${PATH_SLXADMIN}/config.php"
		grep -q 'CONFIG_FOOTER' "${PATH_SLXADMIN}/config.php" || echo "define('CONFIG_FOOTER', '${FOOTER}');" >> "${PATH_SLXADMIN}/config.php"
		echo "$TARGET_WEBIF_VERSION" > "${PATH_SLXADMIN}/version"
		echo "Config updated"
	fi
	# New scripts supplied
	diffcp "slxadmin" "/opt/openslx/slxadmin-cronscript"
	diffcp "slxadmin" "/opt/openslx/slxadmin-bootscript"
	# New crontab supplied
	diffcp "slxadmin" "/etc/cron.d/slx-admin"
	# Service
	diffcp "slxadmin" "/etc/systemd/system/slxadmin-boot.service"
	ln -nfs "../slxadmin-boot.service" "/etc/systemd/system/multi-user.target.wants/slxadmin-boot.service"
	# Enable modules
	(
		mkdir -p "$PATH_SLXADMIN/modules"
		cd "$PATH_SLXADMIN/modules" || perror "Cannot cd to slxadmin"
		echo "Enabling slx-admin modules"
		for mod in adduser backup baseconfig baseconfig_bwidm baseconfig_bwlp bootstrap_datepicker bootstrap_dialog bootstrap_multiselect bootstrap_switch bootstrap_timepicker dnbd3 dozmod eventlog exams js_chart js_circles js_jqueryui js_moment js_selectize js_stupidtable js_vis js_weekcalendar locationinfo locations main minilinux news passthrough permissionmanager rebootcontrol remoteaccess roomplanner runmode serversetup-bwlp-ipxe session statistics statistics_reporting summernote sysconfig syslog systemstatus vmstore webinterface js_ip; do
			name=${mod%%-*}
			rm -f -- "$name"
			ln -s "../modules-available/$mod" "$name" || pwarning "Could not activate module '$mod' (alias '$name')"
		done
		# Update DB
		echo "Running database upgrade"
		cd "$PATH_SLXADMIN"
		if ! ./install-all > "$TMPDIR/wwwup.log"; then
			cat "$TMPDIR/wwwup.log"
			perror "Updating the web interface database failed"
		fi
		if grep -q 'Result: UPDATE_FAILED' "$TMPDIR/wwwup.log"; then
			cat "$TMPDIR/wwwup.log"
			perror "Updating the web interface database failed"
		fi
		if [ "x$(tail -n -1 "$TMPDIR/wwwup.log")" != "xDone." ]; then
			cat "$TMPDIR/wwwup.log"
			pwarning "slxadmin updater didn't finish with 'Done.'"
		fi
	)
	# Delete old statistics api & unneeded dirs
	rm -rf -- "${PATH_SLXADMIN}/apis/statistics.inc.php" "${PATH_SLXADMIN}/doc" "${PATH_SLXADMIN}/.idea" "${PATH_SLXADMIN}/tools" "${PATH_SLXADMIN}/.git"
	# Since 90% is missing by now...
	rm -rf -- "${PATH_SLXADMIN}/lang/pt"
	# Remove link to translation util
	sed -i '/<a.*lang_translations/d' "${PATH_SLXADMIN}/modules-available/main/templates/main-menu.html"

	fixperms "$PATH_SLXADMIN" root:root
	chmod 0640 "$PATH_SLXADMIN/config.php"
	chown root:www-data "$PATH_SLXADMIN/config.php"
	# config.tgz symlink -> db entry
	if [ -L /srv/openslx/www/boot/default/config.tgz ]; then
		echo "Migrating config activation from symlink to database"
		CONFTGZ=$(readlink /srv/openslx/www/boot/default/config.tgz | sed "s/'/\\\'/g")
		echo "config.tgz links to '$CONFTGZ'"
		mysql -e "INSERT IGNORE INTO openslx.configtgz_location (locationid, configid) SELECT 0, configid FROM openslx.configtgz WHERE filepath = '$CONFTGZ' LIMIT 1" \
			|| pwarning "Could not convert default config.tgz setting - do so manually"
		rm -f -- /srv/openslx/www/boot/default/config.tgz
	fi
	ln -s -f "$PATH_TFTP" "/srv/openslx/www/tftp" || perror "Could not symlink $PATH_TFTP to /srv/openslx/www/tftp"
	mkdir -p "${PATH_SLXADMINCACHE}"
	fixperms "${PATH_SLXADMINCACHE}" www-data:www-data

	echo "Symlinking bwIDM slx-admin config"
	rm -f -- "/srv/openslx/www/slx-admin/modules-available/baseconfig_bwidm/baseconfig/settings.json"
	ln -s "${PATH_SLXADMINCACHE}/baseconfig-bwidm_settings.json" /srv/openslx/www/slx-admin/modules-available/baseconfig_bwidm/baseconfig/settings.json || \
		pwarning "Could not set link ${PATH_SLXADMINCACHE}/baseconfig-bwidm_settings.json to /srv/openslx/www/slx-admin/modules-available/baseconfig_bwidm/baseconfig/settings.json - do so manually if you need bwIDM login on client computers"
	# update init script
	rm -rf -- /opt/openslx/restore.d/slxadmin-init
	diffcp "slxadmin" "/opt/openslx/restore.d/slxadmin-init/gpg-key.asc"
	diffcp "slxadmin" "/opt/openslx/restore.d/slxadmin-init/slxadmin-init.php"
	diffcp "slxadmin" "/opt/openslx/restore.d/slxadmin-init/init.sh"
	echo "Web interface upgrade complete"
fi

# ************************* Dozmod *************************
if [ -n "$TGZ_DOZMOD" ]; then
	[ -e "$TMPDIR/$TGZ_DOZMOD" ] || perror "$TGZ_DOZMOD missing from payload."
	echo "* bwLehrpool-Suite Server (dmsd)"
	echo "Adjusting mysql permissions of user sat"
	mysql -e 'GRANT CREATE, ALTER ON sat.* TO sat@localhost' || perror "Could not GRANT permissions ON sat.* to sat@localhost"
	mysql -e 'GRANT SELECT ON openslx.location TO sat@localhost' || perror "Could not GRANT permissions ON openslx.location to sat@localhost"
	echo "Extracting new jar"
	tar -x -C "$PATH_DOZMOD" -f "$TMPDIR/$TGZ_DOZMOD" || perror "Could not extract $TGZ_DOZMOD to $PATH_DOZMOD"
	rm -rf -- "$PATH_DOZMOD/bwLehrpoolSuite_Server.jar" "$PATH_DOZMOD/control.sh" "$PATH_DOZMOD/flag"
	tar -x -C "$PATH_DOZMOD" -f "$TMPDIR/$TGZ_DOZMOD"
	if mysql -e 'SHOW TABLES' openslx | grep -q '^location$'; then
		echo "Enabling location feature"
		sed -i '/^db.location-table\b/d' "$PATH_DOZMOD/config.properties"
		echo 'db.location-table = openslx.location' >> "$PATH_DOZMOD/config.properties"
	fi
	fixperms "$PATH_DOZMOD" root:images
	chmod 0640 "$PATH_DOZMOD/config.properties"
	restart["dmsd.service"]=1
	echo "dmsd upgrade complete"
fi

echo "* Configuring lighttpd"

if (( MAJOR < 10 )); then
	# Filename changed on Debian 10, patch back for 9
	sed -i 's/create-mime\.conf\.pl/create-mime.assign.pl/' "$FILEDIR/lighttpd/etc/lighttpd/lighttpd.conf"
fi
diffcp "lighttpd" "/etc/lighttpd/lighttpd.conf" && restart["lighttpd.service"]=ja
diffcp "lighttpd" "/usr/share/lighttpd/auto-ssl.sh" && restart["lighttpd.service"]=ja
diffcp "lighttpd" "/opt/openslx/slx-cert" && restart["lighttpd.service"]=ja
diffcp "lighttpd" "/usr/share/lighttpd/include-conf-d.sh" && restart["lighttpd.service"]=ja

# ********************** lighttpd php_children *************
mkdir -p /usr/local/sbin

if diffcp "lighttpd" "/usr/local/sbin/patch_lighttpd_phpchildren" "Installing script to set dynamic php children value" \
		|| grep -qF 'patch_lighttpd_phpchildren' "/lib/systemd/system/lighttpd.service"; then
	if [ -f "/lib/systemd/system/lighttpd.service" ]; then
		# remove old hacky way
		sed -i '/patch_lighttpd_phpchildren/d' "/lib/systemd/system/lighttpd.service"
		if ! [ -f "/etc/systemd/system/lighttpd.service.d/10-dynamic_php_children.conf" ]; then
			echo "Adding drop-in to adjust php child count"
			# TODO Unify with installer
			mkdir -p /etc/systemd/system/lighttpd.service.d || perror "Could not create /etc/systemd/system/lighttpd.service.d"
			cat > "/etc/systemd/system/lighttpd.service.d/10-dynamic_php_children.conf" <<-HDOC
				[Service]
				ExecStartPre=/usr/local/sbin/patch_lighttpd_phpchildren
			HDOC
			restart["lighttpd.service"]=ja
		fi
	fi
fi

# ****************** lighttpd extract cert *****************
declare -rg CERT_KEY_FILE="/etc/lighttpd/server.pem"
declare -rg PUB_CERT_FILE="/etc/lighttpd/pub-cert.pem"
if [ -s "$CERT_KEY_FILE" ] && ! [ -s "$PUB_CERT_FILE" ]; then
	echo "Extracting public certificate from server.pem"
	openssl x509 -outform pem -in "$CERT_KEY_FILE" -out "$PUB_CERT_FILE"
fi

# ************************** PHP ***************************

inicount=0
for PHPINIFILE in /etc/php/*/cgi/php.ini; do
	[ -f "$PHPINIFILE" ] || continue
	inicount=$(( inicount + 1 ))
	if ! grep -q -E '^\s*upload_max_filesize = 100M' "$PHPINIFILE"; then
		echo "PHP: Increasing upload_max_filesize to 100MB"
		if grep -q -E '^\s*upload_max_filesize' "$PHPINIFILE"; then
			sed -i -e '/^\s*upload_max_filesize/c\upload_max_filesize = 100M' "$PHPINIFILE" || pwarning "Could not increase PHP upload limit :( (1.a)"
		else
			echo "upload_max_filesize = 100M" >> "$PHPINIFILE" || pwarning "Could not incease PHP upload limit :( (2.a)"
		fi
	fi
	if ! grep -q -E '^\s*post_max_size = 100M' "$PHPINIFILE"; then
		echo "PHP: Increasing post_max_size to 100MB"
		if grep -q -E '^\s*post_max_size' "$PHPINIFILE"; then
			sed -i -e '/^\s*post_max_size/c\post_max_size = 100M' "$PHPINIFILE" || pwarning "Could not increase PHP upload limit :( (1.b)"
		else
			echo "post_max_size = 100M" >> "$PHPINIFILE" || pwarning "Could not incease PHP upload limit :( (2.b)"
		fi
		restart["lighttpd.service"]=ja
	fi
done
if [ "$inicount" -ne 1 ]; then
	pwarning "Odd. Found $inicount php.ini files, expected only one..."
fi

echo "* TFTP"

# ************************** TFTP **************************
if [ -n "$TGZ_TFTP" ]; then
	[ -e "$TMPDIR/$TGZ_TFTP" ] || perror "$TGZ_TFTP missing from payload"
	echo "Extracting tftp server dir contents"
	rm -rf -- "$PATH_TFTP/v4" "$PATH_TFTP/v6" "$PATH_TFTP/ipxelinux.0" "$PATH_TFTP/pxelinux.0"
	mkdir -p "$PATH_TFTP"
	tar -x -C "$PATH_TFTP" -f "$TMPDIR/$TGZ_TFTP" || perror "Could not extract $TGZ_TFTP to $PATH_TFTP"
	fixperms "$PATH_TFTP" taskmanager:taskmanager
	echo "Resetting pxe menu"
	mysql -e 'UPDATE openslx.property SET value = "invalid" WHERE name = "server-ip"' || pwarning "Could not reset pxe menu status; manual regeneration of menu required"
fi

# ************* replace atftpd with tftpd-hpa **************
chg=false
if which "atftpd" &>/dev/null || dpkg-query -W -f='${db:Status-Abbrev}' "atftpd" 2> /dev/null | grep -q '^ii' \
		|| ! which "in.tftpd" &>/dev/null; then
	echo "Replacing atftpd with tftpd-hpa"
	systemctl stop atftpd.service
	failprint apt-get purge -y atftpd || pwarning "Could not uninstall atftpd"
	aptinst tftpd-hpa || perror "Could not install tftpd-hpa. Netboot is now broken."
	chg=true
fi

if [ -e "/etc/systemd/system/atftpd.service" ]; then
	systemctl stop atftpd.service
	rm -f -- "/etc/systemd/system/atftpd.service" "/etc/systemd/system/multi-user.target.wants/atftpd.service"
	chg=true
fi

diffcp "tftpd" "/etc/systemd/system/tftpd-hpa.service" && chg=true
diffcp "tftpd" "/opt/openslx/tftpd-remap" && chg=true
if "$chg"; then
	ln -s -f "../tftpd-hpa.service" "/etc/systemd/system/multi-user.target.wants/"
	daemon_reload=true
	restart["tftpd-hpa.service"]=1
fi

# ********************** dmsd.service **********************
if diffcp "dmsd" "/etc/systemd/system/dmsd.service"; then
	ln -s -f "../dmsd.service" "/etc/systemd/system/multi-user.target.wants/"
	daemon_reload=1
	restart["dmsd.service"]=2
fi

# ********************** dnbd3 config *******************

echo "* Setting up dnbd3"

mkdir -p /etc/dnbd3-server	|| perror "Could not mkdir /etc/dnbd3-server"
mkdir -p "$PATH_DNBD3" || perror "Could not create $PATH_DNBD3"
# check for dnbd3 user: dnbd3, 10002, group dnbd3, 10002

if ! id -u dnbd3 &> /dev/null; then
	dnbd3group=$(getent group dnbd3 2>/dev/null | cut -d ":" -f 3)
	if [ -z "$dnbd3group" ]; then
		dnbd3group=10002
		echo "Creating dnbd3 group with gid $dnbd3group"
		groupadd -g "$dnbd3group" dnbd3 || perror "Could not add dnbd3 group!"
	else
		echo "Using existing group dnbd3 with gid $dnbd3group"
	fi

	echo "Adding user dnbd3"
	useradd --no-create-home -d /nonexistent -s /bin/false --gid "$dnbd3group" --uid 10002 dnbd3 || perror "Could not add user dnbd3!"
fi

# config
diffcp "dnbd3" "/etc/dnbd3-server/server.conf"
diffcp "dnbd3" "/etc/dnbd3-server/rpc.acl"
diffcp "dnbd3" "/etc/dnbd3-server/alt-servers"
rm -f -- "$PATH_DNBD3/is-enabled"

if [ -n "$TGZ_DNBD3" ]; then
	tmpdir="$TMPDIR/dnbd3.tmp"
	mkdir -p "$tmpdir"
	tar --transform='s,^[./]*dnbd3[^/]*/,,' -x -C "$tmpdir" -f "$TMPDIR/$TGZ_DNBD3" || perror "Could not extract $TGZ_DNBD3 to $tmpdir"
	cver=$( "$PATH_DNBD3/dnbd3-server" --version 2> /dev/null | grep -Poi '(?<=Version: ).*(?=, branch)' )
	nver=$( < "$tmpdir/inc/dnbd3/version.h"  grep -Po '(?<=^#define DNBD3_VERSION ").*(?=")' )
	[ -z "$cver" ] && pwarning "BUG BUG! Cannot query dnbd3 version information from currently installed binary!"
	[ -z "$nver" ] && pwarning "BUG BUG! Updater package contains no dnbd3 version information!"
	mkdir -p "$PATH_DNBD3"
	echo "Current: '${cver#v}' | Included: ${nver#v}"
	if [ "${cver#v}" != "${nver#v}" ]; then
		echo "Compiling new dnbd3-server binary..."
		cd "$tmpdir"
		aptinst libjansson-dev
		if (( MAJOR <= 9 )); then
			# cmake too old; try with makefile
			if ! cp "$FILEDIR/dnbd3-Makefile" Makefile; then
				pwarning "Could not get legacy dnbd3 make file"
			elif ! failprint make -j2 dnbd3-server; then
				pwarning "Could not compile new dnbd3-server version $nver"
			elif ! cp -f dnbd3-server "$PATH_DNBD3/dnbd3-server"; then
				pwarning "Successfully built new dnbd3-server, but could not copy binary to $PATH_DNBD3/dnbd3-server"
			else
				cver=
			fi
		else
			# New proper cmake way
			aptinst libfuse-dev
			aptinst cmake
			mkdir build
			cd build
			if ! failprint cmake -DDNBD3_CLIENT_FUSE=OFF -DDNBD3_KERNEL_MODULE=OFF \
					-DDNBD3_SERVER=ON -DDNBD3_SERVER_FUSE=OFF -DCMAKE_BUILD_TYPE=Release ..; then
				pwarning "Could not cmake new version $nver of dnbd3-server"
			elif ! failprint make -j2 dnbd3-server; then
				pwarning "Could not compile new version $nver of dnbd3-server"
			elif ! cp -f src/server/dnbd3-server "$PATH_DNBD3/dnbd3-server"; then
				pwarning "Successfully built new dnbd3-server, but could not copy binary to $PATH_DNBD3/dnbd3-server"
			else # Clear this so we know not to print the "trying to keep going.." message
				cver=
			fi
		fi
		[ -n "$cver" ] && pwarning "Trying to keep going with current version $cver"
	fi
fi

# ******************* dnbd3-server.service ***************
if diffcp "dnbd3" "/etc/systemd/system/dnbd3-server.service"; then
	ln -n -s -f "../dnbd3-server.service" "/etc/systemd/system/multi-user.target.wants/dnbd3-server.service"
	daemon_reload=1
	restart["dnbd3-server.service"]=1
fi

# *************** dnbd3-master-proxy.service *************
if diffcp "dnbd3" "/etc/systemd/system/dnbd3-master-proxy.service"; then
	aptinst socat
	ln -n -s -f "../dnbd3-master-proxy.service" "/etc/systemd/system/multi-user.target.wants/dnbd3-master-proxy.service"
	daemon_reload=2
	restart["dnbd3-master-proxy.service"]=3
fi

# ************************** ldadp *************************
if [ -n "$TGZ_LDADP" ]; then
	[ -e "$TMPDIR/$TGZ_LDADP" ] || perror "$TGZ_LDADP missing from payload"
	echo "* LDAP/AD Proxy"
	tmpdir="$TMPDIR/ldadp.tmp"
	mkdir -p "$tmpdir"
	tar -x -C "$tmpdir" -f "$TMPDIR/$TGZ_LDADP" || perror "Could not extract $TGZ_LDADP to $tmpdir"
	cver=$( /opt/ldadp/ldadp --version 2> /dev/null | awk '{ if ($1 == "Commit:") print $2}' )
	nver=$( awk '{if ($2 == "LDADP_COMMIT") {gsub("\"", "", $3); print $3}}' "$tmpdir/version.in.h" )
	[ -z "$nver" ] && pwarning "BUG BUG! Updater package contains no ldadp version information!"
	mkdir -p "/opt/ldadp"
	if [ "$cver" != "$nver" ]; then
		aptinst libssl-dev
		aptinst libowfat-dev
		echo "Compiling new ldadp binary..."
		cd "$tmpdir"
		if ! failprint make -j2 ldadp; then
			pwarning "Could not make new version $nver of ldadp"
		elif ! cp -f ldadp "$PATH_LDADP/ldadp"; then
			pwarning "Build successful, but could not copy ldadp to $PATH_LDADP"
		else
			cver=
		fi
		[ -n "$cver" ] && pwarning "Trying to keep going with current version $cver"
		killall ldadp 2>/dev/null && pwarning "LDAP/AD Proxy was killed. Client logins might not work for a couple of minutes (see Server Status in webif)"
	fi
	diffcp "ldadp" "/etc/systemd/system/ldadp@.service" && daemon_reload=true
fi

# ************************** tmate ************************
if ! which tmate &> /dev/null && (( MAJOR >= 9 )); then
	echo "* Installing tmate"
	if ! aptinst tmate; then
		pwarning "Could not install tmate. Easy remote assistance will not be available."
	fi
fi
if (( MAJOR >= 11 )); then
	# Fingerprints now need to be SHA256, maybe an old sat got dist-upgraded and still has MD5s in that file
	diffcp "system-base" "/etc/tmux.conf" "Updating tmate config..."
	rm -f -- "/root/.tmate.conf"
fi

# ******************* make sure ssh keypair exists *********
if ! [ -s /root/.ssh/id_rsa ]; then
	echo "* Generating ssh keypair"
	mkdir -p /root/.ssh
	ssh-keygen -f /root/.ssh/id_rsa -N '' -C autogenerated &>/dev/null || pwarning "Could not create new ssh keypair in /root/.ssh/id_rsa"
fi

# Timesync
install_files "timesync"

# Wait for webif callbacks and IPXE compile
echo -n "Finishing."
sudo -u www-data -n php /srv/openslx/www/slx-admin/api.php rebootcontrol --action rebuild >/dev/null 2>&1
for i in 1 1 1 1 2 2 3 4 END; do
	CB=$(sudo -u www-data -n php /srv/openslx/www/slx-admin/api.php cb)
	[ "x$CB" != "xTrue" ] && break
	[ "$i" = "END" ] && break
	echo -n "."
	sleep $i
done
count=0
[ -n "$IPXE_PID" ] && while kill -0 "$IPXE_PID" 2>/dev/null; do
	echo -n "."
	sleep 2
	count=$(( count + 1 ))
	[ "$count" -gt 7 ] && break
done

# ************************** Misc stuff ********************
# This fixes kitfox svg, namely this exception you'd get with
# the headless openjdk install:
# java.awt.AWTError: Assistive Technology not found: org.GNOME.Accessibility.AtkWrapper
sed -i -e '/^assistive_technologies=/s/^/#/' /etc/java-*-openjdk/accessibility.properties &> /dev/null

# Create /var/log/journal to make the journal persistent.
# NOTE: The default journald configuration is set to max 4G space taken
# which should be easily satisfied by the size of the rootfs.
if ! [ -d /var/log/journal ]; then
	mkdir -p /var/log/journal
	systemd-tmpfiles --create --prefix /var/log/journal
	systemctl restart systemd-journald
fi

# Debian 10's default SSL configuration changed to only use DEFAULT@SECLEVEL=2
# as ciphers. We need to patch that, since this can cause problems with ldadp.
if (( MAJOR >= 10 )); then
	sed -r -i 's/^[#\s]*(CipherString =.*)/#\1/' /etc/ssl/openssl.cnf
fi

# ************************** LIGHTY ************************
# XXX
if [ -n "${restart["lighttpd.service"]}" ]; then
	count=0
	if [ -n "$DH_PID" ] && kill -0 "$DH_PID" 2>/dev/null; then
		echo -n ".waiting up to 5 minutes for dh param generation."
		while kill -0 "$DH_PID" 2>/dev/null; do
			echo -n "."
			sleep 2
			count=$(( count + 1 ))
			[ "$count" -gt 150 ] && break
		done
		echo "."
	fi
fi
if [ -n "$daemon_reload" ]; then
	systemctl daemon-reload
fi
for service in "${!restart[@]}"; do
	echo "Restarting $service..."
	restart_service "$service"
done

# Run all post-restore scripts
for i in /opt/openslx/restore.d/*/init.sh; do
	[ -x "$i" ] || continue
	"$i" || pwarning "ERROR running post-restore script $i: $?"
done

# Patch the update version in /etc/motd
sed -r -i 's/(bwLehrpool Satelliten[-sS]*erver, Version[^[(*]*)\s*([[(]Update.*[])])?\s*/\1 [Update: '"${SLXADMIN_FOOTER}"'] /g' /etc/motd

echo ""
echo "Update complete. This update needs a reboot to be working as intended. Please do so now."
echo ""

exit 0
# File end