diff options
author | Simon Rettberg | 2019-07-09 15:51:12 +0200 |
---|---|---|
committer | Simon Rettberg | 2019-07-09 15:51:12 +0200 |
commit | 5fc215ed0cefdab85b543ede8564714e960374ea (patch) | |
tree | e722d4cab86dde6b6d8301b9ad4179a8431a3483 /core/modules | |
parent | [networkd-dispatcher] remove hack (diff) | |
download | mltk-5fc215ed0cefdab85b543ede8564714e960374ea.tar.gz mltk-5fc215ed0cefdab85b543ede8564714e960374ea.tar.xz mltk-5fc215ed0cefdab85b543ede8564714e960374ea.zip |
[dnbd3-proxy-mode] Support local caching
Diffstat (limited to 'core/modules')
4 files changed, 182 insertions, 111 deletions
diff --git a/core/modules/dnbd3-proxy-mode/data/etc/systemd/system/dnbd3-proxy.service b/core/modules/dnbd3-proxy-mode/data/etc/systemd/system/dnbd3-proxy.service index 5896a1db..709e0724 100644 --- a/core/modules/dnbd3-proxy-mode/data/etc/systemd/system/dnbd3-proxy.service +++ b/core/modules/dnbd3-proxy-mode/data/etc/systemd/system/dnbd3-proxy.service @@ -1,5 +1,6 @@ [Unit] Description=DNBD3 proxy server +Requires=setup-dnbd3-proxy.service After=setup-dnbd3-proxy.service [Service] diff --git a/core/modules/dnbd3-proxy-mode/data/etc/systemd/system/dnbd3-tune-network.service b/core/modules/dnbd3-proxy-mode/data/etc/systemd/system/dnbd3-tune-network.service new file mode 100644 index 00000000..a4ec217e --- /dev/null +++ b/core/modules/dnbd3-proxy-mode/data/etc/systemd/system/dnbd3-tune-network.service @@ -0,0 +1,9 @@ +[Unit] +Description=Tweak network stack of %i for maximum throughput +Wants=dhcpc@%i.service +After=dhcpc@%i.service network-interface@%i.service + +[Service] +Type=oneshot +ExecStart=/opt/openslx/scripts/systemd-tweak_network %I + diff --git a/core/modules/dnbd3-proxy-mode/data/etc/systemd/system/multi-user.target.wants/setup-dnbd3-proxy.service b/core/modules/dnbd3-proxy-mode/data/etc/systemd/system/multi-user.target.wants/setup-dnbd3-proxy.service new file mode 120000 index 00000000..ecc3db93 --- /dev/null +++ b/core/modules/dnbd3-proxy-mode/data/etc/systemd/system/multi-user.target.wants/setup-dnbd3-proxy.service @@ -0,0 +1 @@ +../setup-dnbd3-proxy.service
\ No newline at end of file diff --git a/core/modules/dnbd3-proxy-mode/data/opt/openslx/scripts/systemd-setup_dnbd3_proxy b/core/modules/dnbd3-proxy-mode/data/opt/openslx/scripts/systemd-setup_dnbd3_proxy index 5ea076b8..907ab59d 100755 --- a/core/modules/dnbd3-proxy-mode/data/opt/openslx/scripts/systemd-setup_dnbd3_proxy +++ b/core/modules/dnbd3-proxy-mode/data/opt/openslx/scripts/systemd-setup_dnbd3_proxy @@ -6,17 +6,50 @@ ERRLOG="/run/dnbd3-proxy.err" [ -s "$ERRLOG" ] && exit 0 # already ran and failed, don't do it again +. /opt/openslx/bin/slx-tools +. /opt/openslx/config + +# Normally, exit unclean to signal to systemd +islocal= +mingb=10 +if [ "$SLX_RUNMODE_MODULE" != "dnbd3" ]; then + # But if its local proxy mode, always exit 0 since we don't consider this an error + islocal=true + # Not a dnbd3 proxy, see if we have a large partition to try local caching + if [ "$SLX_DNBD3_SERVERS" = "127.0.0.1" ]; then + # Already been here, do nothing + exit 0 + fi + [ "x$SLX_VM_DNBD3" != "xyes" ] && exit 0 # Do not want + [ -z "$SLX_DNBD3_SERVERS" ] && exit 0 # No servers!? + [ -z "$SLX_DNBD3_MIN_GB" ] && exit 0 # Disabled + [ "$SLX_DNBD3_MIN_GB" -ge "$mingb" ] || exit 0 # Disabled + mingb="$SLX_DNBD3_MIN_GB" +fi + # Log error to server and local file in /run. # The main startup script will check whether this file is > 0 bytes # and start in errormsg-mode. errormsg () { echo "$@" >> "$ERRLOG" - slxlog -s -e "dnbd3-setup" "$@" + if [ -z "$islocal" ]; then + slxlog -s -e "dnbd3-setup" "$@" + exit 1 + fi + exit 0 } -# Get size of disk/device at given path, in MB +# Get size of disk/device at given path, in GB disksize () { - df -k "$1" | tail -n 1 | awk '{ print substr($2, 0, length($2)-3) }' + local val len + val="$( fs_path_space "$1" )" + val="${val#* }" + len="${#val}" + if [ "$len" -lt 7 ]; then + echo 0 + else + echo "${val:0:$(( len - 6 ))}" + fi } # Creates the DNBD3 server configuration under DNBD3_CONF_DIR @@ -25,29 +58,25 @@ if ! mkdir -p "${DNBD3_CONF_DIR}"; then errormsg "Failed to create '${DNBD3_CONF_DIR}'." fi -# Checks if the persistent partition (MBR-ID 45 / GPT-LABEL OpenSLX-ID45) -# is present, fallback to /tmp -DNBD3_BASE_DIR="/opt/openslx/persistent" -if grep -q "^/dev/.* ${DNBD3_BASE_DIR} .*rw" /proc/mounts \ - && [ -k "${DNBD3_BASE_DIR}/data" ] \ - && [ "$(disksize "${DNBD3_BASE_DIR}/data")" -gt 12000 ]; then - # setup_partitions creates a data folder in ID45 - # TODO: Partition might be filled with garbage belonging to other users, so - # the usable space for dnbd3 might be much lower. - # avail = "disk size" - "du -s $DNBD3_BASE_DIR" - DNBD3_BASE_DIR="${DNBD3_BASE_DIR}/data" +# We prefer ID45 for storage (if >= 10GB), otherwise +# fallback to /tmp if it's persistent and >= 10GB. +# For local caching, /tmp is never used. +d="/opt/openslx/persistent" +dsd="$(disksize "$d")" +e="/tmp" +dse="$(disksize "$e")" +if fs_path_ismountpoint "$d" && [ -k "$d/data" ] && [ "$dsd" -ge "$mingb" ]; then + DNBD3_BASE_DIR="$d/data" + disksize="$dsd" +elif [ -z "$islocal" ] && ! fs_path_isvolatile "$e" && [ "$dse" -ge "$mingb" ]; then + DNBD3_BASE_DIR="$e" + disksize="$dse" + # We're using /tmp, don't warn about low space + touch "/run/openslx/no-ram-warning" else - # try /tmp fallback - DNBD3_BASE_DIR="/tmp" - if ! grep -q '^/dev/.* '"${DNBD3_BASE_DIR}"' .*rw' /proc/mounts \ - || ! [ "$(disksize "${DNBD3_BASE_DIR}")" -gt 12000 ]; then - # no sane fallback possible - errormsg "Neither a persistent part (ID45) nor /tmp (ID44) are available, or they are < 12GB" - else - # We're using /tmp, don't warn about low space - touch "/run/openslx/no-ram-warning" - fi + errormsg "Neither a persistent part (ID45) nor /tmp (ID44) are available, or they are < ${mingb}GB" fi + # now try to create the actual folder used by the server DNBD3_DATA_DIR="${DNBD3_BASE_DIR}/dnbd3" if ! mkdir -p "${DNBD3_DATA_DIR}"; then @@ -60,7 +89,7 @@ if ! mkdir -p "${DNBD3_DATA_DIR}"; then fi if [ ! -d "${DNBD3_DATA_DIR}" ]; then - errormsg "Failed to create '${DNBD3_BASE_DIR}(.[0-$i])'" + errormsg "Failed to create '${DNBD3_BASE_DIR}/dnbd3' or any of the fallback variants" fi if ! chown -R dnbd3:dnbd3 "${DNBD3_DATA_DIR}"; then errormsg "Failed to chown '${DNBD3_DATA_DIR}' to dnbd3." @@ -71,37 +100,124 @@ fi # Done with sanity checks, now create configs: server.conf & alt-servers # Using the information given by the server in /opt/openslx/config -. /opt/openslx/config -DNBD3_SPARSE="false" -DNBD3_BGR="false" -DNBD3_LOOKUP="false" -DNBD3_SERVER_PENALTY=2000 # no BGR = don't like other servers connecting so much -if [ -n "${SLX_DNBD3_BGR}" ]; then - DNBD3_BGR="true" - DNBD3_SERVER_PENALTY=500 # much better - # Only do chained lookup of image if we're a global proxy with BGR - [ -z "${SLX_DNBD3_WHITELIST}" ] && DNBD3_LOOKUP="true" -fi -if [ -n "${SLX_DNBD3_SPARSE}" ]; then - DNBD3_SPARSE="true" -fi -# Determine if we need sparse mode, or override BGR -cache_size=$(disksize "$DNBD3_DATA_DIR") # MB -if ! [ "$cache_size" -gt 120000 ]; then - # less than 120GiB or NaN, force - [ "$DNBD3_BGR" = "true" ] && slxlog -s -e "dnbd3-setup-small" "Cache partition < 120GiB, forcing background replication OFF and sparse mode ON" +# helper to echo given list of IPs to ${DNBD3_CONF_DIR}/alt-servers +# optionally takes a single char prefix as first param and +# adds it to the IP (for private dnbd3 servers) +# Also sets FOUND_SAT if satellite IP was seen +MY_IPS=$(ip a | grep '^\s*inet\s' | awk '{print $2}') +FOUND_SAT= +add_alt_server() { + local PRE= + if [ "${#1}" = "1" ]; then + PRE="$1" + shift + fi + for ALT in "$@"; do + for ip in $MY_IPS; do + [ "x$ALT" = "x${ip%/*}" ] && return 0 # Ignore self + done + echo "${PRE}${ALT}" >> "${DNBD3_CONF_DIR}/alt-servers" + [ "x${ALT}" = "x${SLX_PXE_SERVER_IP}" ] && FOUND_SAT="oui" + done + return 0 +} + +rm -f -- "${DNBD3_CONF_DIR}/alt-servers" + +# Local caching mode? +if [ -n "$islocal" ]; then + + # Local caching mode. DNBD3_SPARSE="true" - DNBD3_BGR="false" - DNBD3_SERVER_PENALTY=5000 + if [ -n "$SLX_DNBD3_MIN_GB_HASH" ] && [ "$SLX_DNBD3_MIN_GB_HASH" -gt 0 ] && [ "$disksize" -ge "$SLX_DNBD3_MIN_GB_HASH" ]; then + DNBD3_BGR="hashblock" + else + DNBD3_BGR="false" + fi DNBD3_LOOKUP="false" -elif [ "$cache_size" -gt 2000000 ]; then - # force non-sparse on > 2TiB + DNBD3_SERVER_PENALTY="500000" + MAX_REPLICATION_SIZE=500 + + # Use DNBD3 servers from openslx config and then patch it to say localhost + add_alt_server '-' ${SLX_DNBD3_SERVERS} + sed -i "s/^SLX_DNBD3_SERVERS=.*$/SLX_DNBD3_SERVERS='127.0.0.1'/" '/opt/openslx/config' + +else + + # Proxy mode DNBD3_SPARSE="false" -fi + DNBD3_BGR="false" + DNBD3_LOOKUP="false" + DNBD3_SERVER_PENALTY=2000 # no BGR = don't like other servers connecting so much + if [ -n "${SLX_DNBD3_BGR}" ]; then + DNBD3_BGR="true" + DNBD3_SERVER_PENALTY=500 # much better + # Only do chained lookup of image if we're a global proxy with BGR + [ -z "${SLX_DNBD3_WHITELIST}" ] && DNBD3_LOOKUP="true" + fi + if [ -n "${SLX_DNBD3_SPARSE}" ]; then + DNBD3_SPARSE="true" + fi + # Determine if we need sparse mode, or override BGR + if ! [ "$disksize" -gt 120 ]; then + # less than 120GiB or NaN, force + [ "$DNBD3_BGR" = "true" ] && slxlog -s -e "dnbd3-setup-small" "Cache partition < 120GiB, forcing background replication OFF and sparse mode ON" + DNBD3_SPARSE="true" + DNBD3_BGR="false" + DNBD3_SERVER_PENALTY=5000 + DNBD3_LOOKUP="false" + elif [ "$disksize" -gt 2000 ]; then + # force non-sparse on > 2TiB + DNBD3_SPARSE="false" + fi + + MAX_REPLICATION_SIZE=$(( disksize / 6 )) + [ "$MAX_REPLICATION_SIZE" -lt 100 ] && MAX_REPLICATION_SIZE=100 + + add_alt_server ${SLX_DNBD3_PUBLIC} + add_alt_server '-' ${SLX_DNBD3_PRIVATE} + + rm -f "/opt/openslx/iptables/rules.d/99-dnbd3" + # now create iptables helper rules + if [ -n "${SLX_DNBD3_WHITELIST}" ] && [ "${SLX_DNBD3_WHITELIST%/*}" != "${SLX_DNBD3_WHITELIST}" ]; then + # XXX: Remove the second check above after ~ 2018-10-01 -- it's a workaround for broken slx-admin + # that won't properly calculate CIDR notion resulting in a severely locked down proxy :( + DNBD3_IPTABLES_CONF="$(mktemp)" + echo '#!/bin/ash' > "${DNBD3_IPTABLES_CONF}" + for CIDR in ${SLX_DNBD3_WHITELIST} ${SLX_KCL_SERVERS}; do + echo "iptables -I ipt-helper-INPUT 1 -p tcp -s ${CIDR} --dport ${DNBD3_PORT} -j ACCEPT" + done >> "${DNBD3_IPTABLES_CONF}" + echo "iptables -A ipt-helper-INPUT -p tcp --dport ${DNBD3_PORT} -j REJECT" >> "${DNBD3_IPTABLES_CONF}" + chmod +x "${DNBD3_IPTABLES_CONF}" + mv -f "$DNBD3_IPTABLES_CONF" "/opt/openslx/iptables/rules.d/99-dnbd3" + fi -MAX_REPLICATION_SIZE=$(( cache_size / 6 )) -[ "$MAX_REPLICATION_SIZE" -lt 100000 ] && MAX_REPLICATION_SIZE=100000 + ( + echo "*******************" + echo "*** DNBD3 Proxy ***" + echo "*******************" + ) | tee -a "/etc/issue" -a "/etc/issue.net" >> "/opt/openslx/etc/issue.template" + + # Create a crontab for rebooting - if everything is fine, once a weekend, + # on failure, reboot every night, hoping things will get better (...) + M=$(( RANDOM % 60 )) + H=$(( RANDOM % 5 )) + if [ -s "$ERRLOG" ]; then + W="*" + else + W="0" + fi + cat > "/etc/cron.d/dnbd3-reboot" <<EOF +# OpenSLX: Reboot proxy at night +SHELL=/bin/ash +PATH=/usr/sbin:/usr/bin:/sbin:/bin:/opt/openslx/sbin:/opt/openslx/bin + +$M $H * * $W root reboot +EOF + # + +fi DNBD3_PORT=5003 rm -f "${DNBD3_CONF_DIR}/server.conf" @@ -121,35 +237,12 @@ uplinkTimeout=5000 clientTimeout=15000 [limits] -maxReplicationSize=${MAX_REPLICATION_SIZE}M +maxReplicationSize=${MAX_REPLICATION_SIZE}G [logging] consoleMask=ERROR WARNING MINOR INFO EOF -MY_IPS=$(ip a | grep '^\s*inet\s' | awk '{print $2}') - -# helper to echo given list of IPs to ${DNBD3_CONF_DIR}/alt-servers -# optionally takes a single char prefix as first param and -# adds it to the IP (for private dnbd3 servers) -# Also sets FOUND_SAT if satellite IP was seen -add_alt_server() { - local PRE= - [ "x$1" = "x-" ] && PRE='-' && shift - for ALT in "$@"; do - for ip in $MY_IPS; do - [ "x$ALT" = "x${ip%/*}" ] && return 0 # Ignore self - done - echo "${PRE}${ALT}" >> "${DNBD3_CONF_DIR}/alt-servers" - [ "x${ALT}" = "x${SLX_PXE_SERVER_IP}" ] && FOUND_SAT="oui" - done - return 0 -} - -rm -f "${DNBD3_CONF_DIR}/alt-servers" -FOUND_SAT= -add_alt_server ${SLX_DNBD3_PUBLIC} -add_alt_server '-' ${SLX_DNBD3_PRIVATE} # To this day, only the sat IP is in SLX_KCL_SERVERS afaik [ -z "${FOUND_SAT}" ] && add_alt_server ${SLX_KCL_SERVERS} chmod -R a+Xr "${DNBD3_CONF_DIR}" @@ -160,43 +253,10 @@ for SRV in ${SLX_KCL_SERVERS}; do echo "$SRV ALL" >> "${DNBD3_CONF_DIR}/rpc.acl" done -rm -f "/opt/openslx/iptables/rules.d/99-dnbd3" -# now create iptables helper rules -if [ -n "${SLX_DNBD3_WHITELIST}" ] && [ "${SLX_DNBD3_WHITELIST%/*}" != "${SLX_DNBD3_WHITELIST}" ]; then - # XXX: Remove the second check above after ~ 2018-10-01 -- it's a workaround for broken slx-admin - # that won't properly calculate CIDR notion resulting in a severely locked down proxy :( - DNBD3_IPTABLES_CONF="$(mktemp)" - echo '#!/bin/ash' > "${DNBD3_IPTABLES_CONF}" - for CIDR in ${SLX_DNBD3_WHITELIST} ${SLX_KCL_SERVERS}; do - echo "iptables -I ipt-helper-INPUT 1 -p tcp -s ${CIDR} --dport ${DNBD3_PORT} -j ACCEPT" - done >> "${DNBD3_IPTABLES_CONF}" - echo "iptables -A ipt-helper-INPUT -p tcp --dport ${DNBD3_PORT} -j REJECT" >> "${DNBD3_IPTABLES_CONF}" - chmod +x "${DNBD3_IPTABLES_CONF}" - mv -f "$DNBD3_IPTABLES_CONF" "/opt/openslx/iptables/rules.d/99-dnbd3" -fi - -( - echo "*******************" - echo "*** DNBD3 Proxy ***" - echo "*******************" -) | tee -a "/etc/issue" | tee -a "/etc/issue.net" >> "/opt/openslx/etc/issue.template" - -# Create a crontab for rebooting - if everything is fine, once a weekend, -# on failure, reboot every night, hoping things will get better (...) -M=$(( RANDOM % 60 )) -H=$(( RANDOM % 5 )) -if [ -s "$ERRLOG" ]; then - W="*" -else - W="0" +if [ -n "$islocal" ]; then + # Start service, is not symlinked when not in proxy mode + systemctl --no-block start dnbd3-proxy.service fi -cat > "/etc/cron.d/dnbd3-reboot" <<EOF -# OpenSLX: Reboot proxy at night -SHELL=/bin/ash -PATH=/usr/sbin:/usr/bin:/sbin:/bin:/opt/openslx/sbin:/opt/openslx/bin - -$M $H * * $W root reboot -EOF exit 0 |