restructured

'builder' builds the initramfs with our dracut module
'packager' builds/packs the stage4 rootfs

15 files changed, 335 insertions, 0 deletions

diff --git a/packager/blacklists/README b/packager/blacklists/README
new file mode 100644
index 00000000..ed3fe459
--- /dev/null
+++ b/packager/blacklists/README
@@ -0,0 +1,15 @@
+This directory contains blacklists of files and directories that
+should be excluded when cloning stage4 of a remote system. The
+blacklists are split into groups, represented by sub-directories.
+They contain one or more text files representing blacklists in rsync
+"--exclude-from" format.
+
+There is a group called "essential", containg blacklists for essential
+files and dirs that you always want to exclude from being cloned.
+Other directories are create for specific software packages, like
+gnome, or kde.
+
+For inclusion of desktop environments like Gnome or KDE a substructure
+(subdirectory) called desktop-sessions was introduced. The files /
+directories in there should describe only includes ("+ ....") to provide
+files needed for a desktop environment.
diff --git a/packager/blacklists/essential/fontcache b/packager/blacklists/essential/fontcache
new file mode 100644
index 00000000..13d60d44
--- /dev/null
+++ b/packager/blacklists/essential/fontcache
@@ -0,0 +1,3 @@
++ /var/cache/fontconfig
++ /var/cache/fontconfig/*
+
diff --git a/packager/blacklists/essential/linux-base b/packager/blacklists/essential/linux-base
new file mode 100644
index 00000000..c3d023d7
--- /dev/null
+++ b/packager/blacklists/essential/linux-base
@@ -0,0 +1,80 @@
+- /cdrom/*
+- /dev/*
+- /adm/*
+- /afs/*
+- /bfg/*
+- /clustersw/*
+- /cvmfs/*
+- /gridsw/*
+- /storage/*
+
+# This is where the bind-mount of mltk resides...
+- /export/*
+- /etc/*ppp*
+- /etc/*pptp*
+- /etc/dhcp*
+- /etc/exports*
+- /etc/fstab
+#- /etc/group
+- /etc/group-
+- /etc/gshadow
+- /etc/gshadow-
+- /etc/hostname
+- /etc/hosts
+#- /etc/inputrc
+- /etc/issue.net
+# Always include ldconfig/cache of full system:
++ /etc/ld.so.**
+#- /etc/localtime
+#- /etc/login.defs
+#- /etc/mtab
+#- /etc/mtab.*
++ /etc/sysconfig/network-scripts/ifcfg-lo
+- /etc/sysconfig/network-scripts/ifcfg-*
+- /etc/networks
+#- /etc/passwd
+- /etc/passwd-
+#- /etc/resolv.conf
+#- /etc/resolvconf/
+#- /etc/shadow
+- /etc/shadow-
+#- /lib/modules/
+- /lost+found/
+- /media/*
+- /mnt/*
+- /opt/openslx
+- /proc/*
+- /run/*
+#- /srv/
+- /sys/*
+- /tmp/*
+#- /usr/src/
++ /var/backup/**/
+- /var/backup/**
++ /var/cache/**/
+- /var/cache/**
++ /var/crash/**/
+- /var/crash/**
++ /var/empty/**/
+- /var/empty/**
++ /var/lock/**/
+- /var/lock/**
++ /var/log/**/
+- /var/log/**
++ /var/mail/**/
+- /var/mail/**
+#+ /var/spool/**/
+#- /var/spool/**
++ /var/spool/torque/*
+- /var/tmp/*
+- /var/lib/nova/instances/*
+- /var/lib/libvirt
+
+
+
+## Interfering binaries/links
+#- /sbin/shutdown
+#- /sbin/reboot
+#- /sbin/poweroff
+#- /sbin/halt
+##- /bin/sh
diff --git a/packager/blacklists/essential/linux-extended b/packager/blacklists/essential/linux-extended
new file mode 100644
index 00000000..3d81000a
--- /dev/null
+++ b/packager/blacklists/essential/linux-extended
@@ -0,0 +1,5 @@
+- /home/*
+- /root/*
++ /root/.bash*
++ /root/.ssh
+- /etc/cups
diff --git a/packager/blacklists/essential/other-files b/packager/blacklists/essential/other-files
new file mode 100644
index 00000000..d35a5736
--- /dev/null
+++ b/packager/blacklists/essential/other-files
@@ -0,0 +1,3 @@
+- .svn
+- .git
+- *.gvfs
diff --git a/packager/blacklists/essential/policykit b/packager/blacklists/essential/policykit
new file mode 100644
index 00000000..2f42c0e0
--- /dev/null
+++ b/packager/blacklists/essential/policykit
@@ -0,0 +1,2 @@
+#- /usr/share/pol*/actions/org.freedesktop.upower.*
+
diff --git a/packager/blacklists/essential/temp-files b/packager/blacklists/essential/temp-files
new file mode 100644
index 00000000..50955d08
--- /dev/null
+++ b/packager/blacklists/essential/temp-files
@@ -0,0 +1,5 @@
+- *~
+- *.bak
+- *.old
+- *.pid
+- *.tmp
diff --git a/packager/blacklists/essential/tm-scripts b/packager/blacklists/essential/tm-scripts
new file mode 100644
index 00000000..393effea
--- /dev/null
+++ b/packager/blacklists/essential/tm-scripts
@@ -0,0 +1 @@
+- **/tm-scripts
diff --git a/packager/blacklists/package-managers/apt b/packager/blacklists/package-managers/apt
new file mode 100644
index 00000000..5e019138
--- /dev/null
+++ b/packager/blacklists/package-managers/apt
@@ -0,0 +1,2 @@
++ /var/cache/apt/archives/partial/**
+
diff --git a/packager/blacklists/package-managers/rpm b/packager/blacklists/package-managers/rpm
new file mode 100644
index 00000000..0527ee75
--- /dev/null
+++ b/packager/blacklists/package-managers/rpm
@@ -0,0 +1,5 @@
+- *.rpmnew
+- *.rpmsave
+- /etc/*rpmnew
+- /etc/*rpmorig
+- /etc/*rpmsave
diff --git a/packager/blacklists/package-managers/yast b/packager/blacklists/package-managers/yast
new file mode 100644
index 00000000..9de1ec05
--- /dev/null
+++ b/packager/blacklists/package-managers/yast
@@ -0,0 +1,5 @@
+- /var/lib/YaST2/you/mnt/*
+- /var/lib/YaST2/backup_boot_sectors
+- /etc/*YaST2save
+- *.YaST2save
+
diff --git a/packager/blacklists/package-managers/yum b/packager/blacklists/package-managers/yum
new file mode 100644
index 00000000..8b137891
--- /dev/null
+++ b/packager/blacklists/package-managers/yum
@@ -0,0 +1 @@
+
diff --git a/packager/blacklists/package-managers/zypper b/packager/blacklists/package-managers/zypper
new file mode 100644
index 00000000..fbab85d7
--- /dev/null
+++ b/packager/blacklists/package-managers/zypper
@@ -0,0 +1,2 @@
+- /var/lib/zypp
+
diff --git a/packager/openslx b/packager/openslx
new file mode 100755
index 00000000..6a75fc47
--- /dev/null
+++ b/packager/openslx
@@ -0,0 +1,68 @@
+#!/bin/bash
+# ------------------------------------------------------------------------------
+#
+#                        OpenSLX Toolkit Next Gen
+#
+# ------------------------------------------------------------------------------
+#
+# The next generation toolkit to build a stateless linux system bootable
+# over the network. Composed of dracut-built initramfs and a full system clone.
+#
+# ------------------------------------------------------------------------------
+#
+#                           OpenSLX-NG Functions
+#
+# ------------------------------------------------------------------------------
+pinfo() {
+	echo -e "\033[38;5;10m[info]\033[0m $@"
+}
+
+perror() {
+	echo -e "\033[38;5;9m[error]\033[0m $@"
+	kill "$SELF_PID"
+	exit 1
+}
+
+print_usage() {
+	perror "$ARG0 <remote_ip> <stage4_sync_dir> <target_qcow2_container>"
+}
+
+# ------------------------------------------------------------------------------
+#
+#                           OpenSLX-NG Main Code
+#
+# ------------------------------------------------------------------------------
+
+declare -rg ARG0="$0"
+declare -rg SELF="$(readlink -f "$ARG0")"
+declare -rg ROOT_DIR="$(dirname "${SELF}")"
+declare -rg SELF_PID="$$"
+
+# root check
+if [ "$(id -u)" -ne 0 ]; then
+	perror "ERROR: You need to be root to use this toolkit."
+else
+	pinfo ""
+fi
+
+# do we even have tools?
+for TOOL in qemu-img qemu-nbd mkfs.ext4; do
+	which $TOOL &>/dev/null || perror "Could not find '$TOOL'."
+done
+
+# let's keep it simple for now, just source our main functions file
+. "${ROOT_DIR}/$ARG0".functions || perror "Could not source functions."
+
+# ok, now check that we have 3 args
+[ $# -ne 3 ] && print_usage
+
+# even though functions check their parameter, we also do it to be safe.
+valid_ip "$1" || perror "'$1' is not a valid IP adress, aborting..."
+[ ! -d "$2" ] || perror "'$2' already exists, aborting..."
+[ ! -f "$3" ] || perror "'$3' already exists, aborting..."
+
+# all good, let's start
+pinfo " ## RUNNING ##"
+clone_stage4 $1 $2 || perror "Failed to clone stage4 with: $1 $2"
+export_qcow2 $2 $3 || perror "Failed to export stage4 with: $2 $3"
+pinfo " ## COMPLETE ##"
diff --git a/packager/openslx.functions b/packager/openslx.functions
new file mode 100755
index 00000000..5fabdbe0
--- /dev/null
+++ b/packager/openslx.functions
@@ -0,0 +1,138 @@
+# ------------------------------------------------------------------------------
+#
+#                          Functions for OpenSLX-NG
+#
+# ------------------------------------------------------------------------------
+#
+# check if we have our environement variables to check
+# if we actually got sourced by the main script
+if [ -z "${SELF_PID}" -o  -z "${ROOT_DIR}" ]; then
+	# not using perror, since we probably don't have it.
+	echo "Neither SELF_PID nor ROOT_DIR is set. Was this included by OpenSLX-NG?"
+	exit 1
+fi
+
+# Helper to generate a stage4 export for a remote machine per rsync.
+clone_stage4() {
+	[ $# -ne 2 ] && perror "$0 <remote_ip> <stage4_sync_dir>"
+	local REMOTE_HOST="$1"
+	local TARGET_DIR="$2"
+	local BUILD_DIR="${ROOT_DIR}/builds/$REMOTE_HOST"
+	mkdir -p "$TARGET_DIR"
+	mkdir -p "$BUILD_DIR"
+
+	local EXCLUDE="$BUILD_DIR/exclude-stage4"
+	local INCLUDE="$BUILD_DIR/include-stage4"
+
+	pinfo "Building rsync exclude-file for building stage 4...."
+	echo "## Exclude file for stage4 of $REMOTE_HOST" > "$EXCLUDE"
+	echo "## Include file for stage4 of $REMOTE_HOST" > "$INCLUDE"
+	for FILE in $(find "$ROOT_DIR"/blacklists/*/ -type f); do
+		echo "## From $FILE" >> "$EXCLUDE"
+		echo "## From $FILE" >> "$INCLUDE"
+		grep '^-' "$FILE" >> "$EXCLUDE"
+		grep '^+' "$FILE" >> "$INCLUDE"
+	done
+	pinfo "Done."
+
+	# prepare rsync's options
+	local RSYNC_SOURCE="root@$REMOTE_HOST:/"
+	local RSYNC_OPTS="-e ssh -c arcfour -oStrictHostKeyChecking=no"
+
+	# run rsync with the exclude/include lists created earlier
+	pinfo "Cloning via rsync..."
+	cat "$INCLUDE" "$EXCLUDE" | \
+		rsync --verbose \
+				--archive \
+				--delete \
+				--delete-excluded \
+				--numeric-ids \
+				--exclude-from=- \
+				"${RSYNC_OPTS}" \
+				"${RSYNC_SOURCE}" \
+				"${TARGET_DIR}" \
+		|| perror "rsync from '${RSYNC_SOURCE}' to '${TARGET_DIR}' failed."
+	pinfo "Cloning '${REMOTE_HOST}' to '${TARGET_DIR}' succeeded."
+	return 0
+}
+# helper to build a qcow2 container from a stage4 sync directory.
+# required tools: qemu-img, qemu-nbd, nbd, mkfs.ext4
+export_qcow2() {
+	# enough args?
+	[ $# -ne 2 ] && perror "Usage: $0 <stage4_synced_dir> <target_file>"
+	# $1 valid?
+	[ ! -d $1 ] && perror "First argument not a directory!"
+	# $2 should not be a dir or strange things will happen...
+	[ -d $2 ] && perror "Target file can not be a directory!"
+
+	# $1 is dir, strip trailing slash if there is one
+	local STAGE4_DIR="${1%/}"
+	local TARGET_FILE="$2"
+	if [ -e $TARGET_FILE ]; then
+		pinfo "Removing old target: $TARGET_FILE"
+		rm $target || perror "Could not remove '$TARGET_FILE'"
+	fi
+
+	# so far so good
+	pinfo "Creating empty qcow2-container ..."
+	qemu-img create -f qcow2 $TARGET_FILE 10G \
+		|| perror "qemu-img create failed with: $?"
+	pinfo "Done."
+
+	# find usable nbd device
+	pinfo "Looking for usable nbd device..."
+	local NBD_ID="$(find_free_nbd)"
+	local NBD_DEV="/dev/nbd${NBD_ID}"
+	[ -b "$NBD_DEV" ] || perror "'$NBD_DEV' is not a block device!"
+	pinfo "Exporting '${TARGET_FILE}' using '${NBD_DEV}'..."
+	qemu-nbd -c ${NBD_DEV} ${TARGET_FILE} || perror "qemu-nbd failed with: $?"
+	pinfo "Done."
+
+	pinfo "Creating ext4 filesystem on '${TARGET_FILE}'..."
+	mkfs.ext4 "${NBD_DEV}" || perror "mkfs failed with: $?"
+	pinfo "Done."
+
+	# mount NBD_DEV
+	mkdir ${BUILD_DIR}/mnt
+	pinfo "Mounting '${NBD_DEV}' to '${BUILD_DIR}/mnt'..."
+	mount ${NBD_DEV} ${BUILD_DIR}/mnt || perror "Mount failed with: $?"
+	pinfo "Done."
+
+	# copy files from the stage4 directory to the mounted qcow2-container
+	pinfo "Copying '${STAGE4_DIR}' to '${BUILD_DIR}/mnt'..."
+	cp -ra "${STAGE4_DIR}"/* "${BUILD_DIR}"/mnt || perror "Copying failed with: $?"
+	pinfo "Done."
+
+	pinfo "Cleaning up..."
+	umount "${BUILD_DIR}/mnt" || perror "Could not unmount '${BUILD_DIR}/mnt'."
+	qemu-nbd -d /dev/nbd${nbd_id} || perror "Could not disconnect '${NBD_DEV}'."
+	pinfo "Exporting '${STAGE4_DIR}' to '${TARGET_FILE}' completed."
+}
+
+# helper to find an unused nbd device
+find_free_nbd() {
+	local nbd_size=0
+	for nbd_id in {0..15}; do
+		[ -b "/dev/nbd${nbd_id}" ] || continue
+		[ ! -e "/sys/block/nbd${nbd_id}/size" ] || continue
+		nbd_size=$(cat /sys/block/nbd${nbd_id}/size)
+		[ $nbd_size -eq 0 ] && break
+	done
+	return ${nbd_id}
+}
+# helper to validate an ip
+valid_ip() {
+    local  ip=$1
+    local  stat=1
+
+    if [[ $ip =~ ^[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}$ ]]; then
+        OIFS=$IFS
+        IFS='.'
+        ip=($ip)
+        IFS=$OIFS
+        [[ ${ip[0]} -le 255 && ${ip[1]} -le 255 \
+            && ${ip[2]} -le 255 && ${ip[3]} -le 255 ]]
+        stat=$?
+    fi
+    return $stat
+}