From 3f9904d6a73d7795b102553cc94ec9ecce9d0b1c Mon Sep 17 00:00:00 2001 From: Simon Rettberg Date: Fri, 7 Jun 2024 16:20:59 +0200 Subject: [shib_secondary] Tool to deploy a secondary masterserver as fallback --- shib_secondary/deploy.sh | 75 +++++++++++++++++++ shib_secondary/remote/000-bwlp-default.conf | 11 +++ shib_secondary/remote/110-bwlp-active.conf | 59 +++++++++++++++ shib_secondary/remote/110-bwlp-passthrough.conf | 58 +++++++++++++++ shib_secondary/remote/alt-servers | 4 + shib_secondary/remote/bwlp-check-master.service | 6 ++ shib_secondary/remote/bwlp-check-master.timer | 10 +++ shib_secondary/remote/bwlp-copy-config.service | 6 ++ shib_secondary/remote/bwlp-copy-config.timer | 10 +++ shib_secondary/remote/bwlp-master-socat@.service | 10 +++ shib_secondary/remote/bwlp-master.service | 11 +++ shib_secondary/remote/check-and-toggle-mode.sh | 92 +++++++++++++++++++++++ shib_secondary/remote/config.php | 36 +++++++++ shib_secondary/remote/copy-current-config.sh | 86 ++++++++++++++++++++++ shib_secondary/remote/dnbd3-server.service | 18 +++++ shib_secondary/remote/dnbd3-ssl-in.service | 10 +++ shib_secondary/remote/dnbd3-ssl-out.service | 11 +++ shib_secondary/remote/global.properties | 45 ++++++++++++ shib_secondary/remote/install.sh | 94 ++++++++++++++++++++++++ shib_secondary/remote/mysql.properties | 4 + shib_secondary/remote/server.conf | 25 +++++++ 21 files changed, 681 insertions(+) create mode 100755 shib_secondary/deploy.sh create mode 100644 shib_secondary/remote/000-bwlp-default.conf create mode 100644 shib_secondary/remote/110-bwlp-active.conf create mode 100644 shib_secondary/remote/110-bwlp-passthrough.conf create mode 100644 shib_secondary/remote/alt-servers create mode 100644 shib_secondary/remote/bwlp-check-master.service create mode 100644 shib_secondary/remote/bwlp-check-master.timer create mode 100644 shib_secondary/remote/bwlp-copy-config.service create mode 100644 shib_secondary/remote/bwlp-copy-config.timer create mode 100644 shib_secondary/remote/bwlp-master-socat@.service create mode 100644 shib_secondary/remote/bwlp-master.service create mode 100755 shib_secondary/remote/check-and-toggle-mode.sh create mode 100644 shib_secondary/remote/config.php create mode 100755 shib_secondary/remote/copy-current-config.sh create mode 100644 shib_secondary/remote/dnbd3-server.service create mode 100644 shib_secondary/remote/dnbd3-ssl-in.service create mode 100644 shib_secondary/remote/dnbd3-ssl-out.service create mode 100644 shib_secondary/remote/global.properties create mode 100755 shib_secondary/remote/install.sh create mode 100644 shib_secondary/remote/mysql.properties create mode 100644 shib_secondary/remote/server.conf diff --git a/shib_secondary/deploy.sh b/shib_secondary/deploy.sh new file mode 100755 index 0000000..536f221 --- /dev/null +++ b/shib_secondary/deploy.sh @@ -0,0 +1,75 @@ +#!/bin/bash + +destination= +domain= +master= +secret= + +while (( $# > 0 )); do + case "$1" in + --dest*) + destination="$2" + shift + ;; + --domain) + domain="$2" + shift + ;; + --master) + master="$2" + shift + ;; + --secret) + secret="$2" + shift + ;; + *) + echo "WAAAAT? NO MAHNEEY?" + exit 1 + esac + shift +done + +if ! [[ $destination =~ ^[a-z0-9_]+@[a-z0-9_.-]+$ ]] \ + || [ -z "$domain" ] \ + || [ -z "$secret" ] \ + || ! [[ "$master" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "Usage: $0 --dest user@4.5.6.7 --domain foo.bar.example.com --master 1.2.3.4 --secret your_master_secret" + exit 1 +fi + +if [[ $destination == root@* ]]; then + sudo= +else + echo "+++++++++++++++++++++++++++++++++++++++++++++++++++++" + echo "Using sudo on the remote server to gain root privs..." + sudo="sudo " +fi + +echo + +if ! ssh "$destination" "$sudo rm -rf -- /tmp/shib_deploy"; then + echo "Remote access for preparation failed :-(" + exit 1 +fi + +if ! scp -r "$( dirname -- "${BASH_SOURCE[0]}" )/remote" "${destination}:/tmp/shib_deploy"; then + echo "Copying data to $destination failed" + exit 1 +fi + +if ! ssh "$destination" "$sudo /tmp/shib_deploy/install.sh --domain '$domain' --master '$master' --secret '$secret'"; then + echo "Remote install failed :-(" + exit 1 +fi + +echo "------------------------------------------------------" +echo +echo "Success" +echo +echo +echo "Remember to make sure the new R/O instance's IP address" +echo "is allowed to pull the data from $master, and added to" +echo "the trusted proxy IP addresses." +echo "(RemoteIPInternalProxy)" +echo diff --git a/shib_secondary/remote/000-bwlp-default.conf b/shib_secondary/remote/000-bwlp-default.conf new file mode 100644 index 0000000..4c7b955 --- /dev/null +++ b/shib_secondary/remote/000-bwlp-default.conf @@ -0,0 +1,11 @@ + + ServerName foobar + ServerAdmin webmaster@localhost + Redirect permanent / https://%DOMAIN%/ + + # Possible values include: debug, info, notice, warn, error, crit, + # alert, emerg. + LogLevel warn + CustomLog ${APACHE_LOG_DIR}/access.log combined + ErrorLog ${APACHE_LOG_DIR}/error.log + diff --git a/shib_secondary/remote/110-bwlp-active.conf b/shib_secondary/remote/110-bwlp-active.conf new file mode 100644 index 0000000..590fc4f --- /dev/null +++ b/shib_secondary/remote/110-bwlp-active.conf @@ -0,0 +1,59 @@ + + ServerName %DOMAIN% + + UseCanonicalName On + + DocumentRoot /var/www/masterserver + + + Options FollowSymLinks + AllowOverride None + + + + Require all granted + AuthType shibboleth + ShibRequestSetting requireSession false + require shibboleth + + + + AuthType None + Require all granted + + + AuthType None + Require all granted + + + Alias /shibboleth-sp/main.css /usr/share/shibboleth/main.css + Alias /shibboleth-sp/logo.jpg /usr/share/shibboleth/logo.jpg + + + AuthType shibboleth + ShibRequestSetting requireSession true + require valid-user + + + # optional (Metadata-Access at entityID-URL) + Redirect seeother /shibboleth /Shibboleth.sso/Metadata + RedirectMatch /start-session$ /Shibboleth.sso/Login + + LogLevel warn + ErrorLog ${APACHE_LOG_DIR}/masterserver/error.log + CustomLog ${APACHE_LOG_DIR}/masterserver/access.log combined + + SSLEngine on + SSLOptions +StrictRequire + # HSTS (mod_headers is required) (15768000 seconds = 6 months) + Header always set Strict-Transport-Security "max-age=15768000" + + SSLCertificateFile /opt/bwlp/ssl/live/ssl-cert/cert.pem + SSLCertificateKeyFile /opt/bwlp/ssl/live/ssl-cert/privkey.pem + SSLCertificateChainFile /opt/bwlp/ssl/live/ssl-cert/chain.pem + + + SSLOptions +StdEnvVars + + + diff --git a/shib_secondary/remote/110-bwlp-passthrough.conf b/shib_secondary/remote/110-bwlp-passthrough.conf new file mode 100644 index 0000000..807d8f8 --- /dev/null +++ b/shib_secondary/remote/110-bwlp-passthrough.conf @@ -0,0 +1,58 @@ + + ServerName %DOMAIN% + + UseCanonicalName On + + DocumentRoot /var/www/masterserver + + + Options FollowSymLinks + AllowOverride None + + + + Require all granted + AuthType shibboleth + ShibRequestSetting requireSession false + require shibboleth + + + + AuthType None + Require all granted + + + AuthType None + Require all granted + + + Alias /shibboleth-sp/main.css /usr/share/shibboleth/main.css + Alias /shibboleth-sp/logo.jpg /usr/share/shibboleth/logo.jpg + + SSLProxyEngine on + + ProxyPassMatch "https://%DOMAIN%/$1$2" connectiontimeout=5 timeout=30 max=10 + + + # optional (Metadata-Access at entityID-URL) + Redirect seeother /shibboleth /Shibboleth.sso/Metadata + RedirectMatch /start-session$ /Shibboleth.sso/Login + + LogLevel warn + ErrorLog ${APACHE_LOG_DIR}/masterserver/error.log + CustomLog ${APACHE_LOG_DIR}/masterserver/access.log combined + + SSLEngine on + SSLOptions +StrictRequire + # HSTS (mod_headers is required) (15768000 seconds = 6 months) + Header always set Strict-Transport-Security "max-age=15768000" + + SSLCertificateFile /opt/bwlp/ssl/live/ssl-cert/cert.pem + SSLCertificateKeyFile /opt/bwlp/ssl/live/ssl-cert/privkey.pem + SSLCertificateChainFile /opt/bwlp/ssl/live/ssl-cert/chain.pem + + + SSLOptions +StdEnvVars + + + diff --git a/shib_secondary/remote/alt-servers b/shib_secondary/remote/alt-servers new file mode 100644 index 0000000..5c8bebc --- /dev/null +++ b/shib_secondary/remote/alt-servers @@ -0,0 +1,4 @@ +[127.0.0.1:5005] +comment=SSL tunnel to %DOMAIN%:5006 for MaxiLinux +for=replication +namespace=stage4/bwlp/ diff --git a/shib_secondary/remote/bwlp-check-master.service b/shib_secondary/remote/bwlp-check-master.service new file mode 100644 index 0000000..f8528db --- /dev/null +++ b/shib_secondary/remote/bwlp-check-master.service @@ -0,0 +1,6 @@ +[Unit] +Description=Check availability of master server and enable/disable fallback mode + +[Service] +Type=oneshot +ExecStart=/opt/bwlp/check-and-toggle-mode.sh diff --git a/shib_secondary/remote/bwlp-check-master.timer b/shib_secondary/remote/bwlp-check-master.timer new file mode 100644 index 0000000..e2b83e5 --- /dev/null +++ b/shib_secondary/remote/bwlp-check-master.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Trigger availability check of masterserver + +[Timer] +OnBootSec=10 +OnUnitActiveSec=120 +AccuracySec=10 + +[Install] +WantedBy=timers.target diff --git a/shib_secondary/remote/bwlp-copy-config.service b/shib_secondary/remote/bwlp-copy-config.service new file mode 100644 index 0000000..a4ad6c5 --- /dev/null +++ b/shib_secondary/remote/bwlp-copy-config.service @@ -0,0 +1,6 @@ +[Unit] +Description=Download current config from masterserver + +[Service] +Type=oneshot +ExecStart=/opt/bwlp/copy-current-config.sh diff --git a/shib_secondary/remote/bwlp-copy-config.timer b/shib_secondary/remote/bwlp-copy-config.timer new file mode 100644 index 0000000..96cea94 --- /dev/null +++ b/shib_secondary/remote/bwlp-copy-config.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Trigger download of current config + +[Timer] +OnBootSec=2m +OnUnitActiveSec=12h +AccuracySec=10m + +[Install] +WantedBy=timers.target diff --git a/shib_secondary/remote/bwlp-master-socat@.service b/shib_secondary/remote/bwlp-master-socat@.service new file mode 100644 index 0000000..4e82880 --- /dev/null +++ b/shib_secondary/remote/bwlp-master-socat@.service @@ -0,0 +1,10 @@ +[Unit] +Description=Tunnel TCP:%i connections to actual masterserver + +[Service] +Type=simple +Restart=always +ExecStart=/usr/bin/socat tcp-listen:%i,reuseaddr,su=nobody,fork tcp:%DOMAIN%:%i + +[Install] +WantedBy=multi-user.target diff --git a/shib_secondary/remote/bwlp-master.service b/shib_secondary/remote/bwlp-master.service new file mode 100644 index 0000000..d0e1d20 --- /dev/null +++ b/shib_secondary/remote/bwlp-master.service @@ -0,0 +1,11 @@ +[Unit] +Description=bwLehrpool Masterserver + +[Service] +User=bwlp +WorkingDirectory=/home/bwlp/server +ExecStart=/usr/bin/java -Dlog4j2.formatMsgNoLookups=true -Xmx2G -jar /home/bwlp/server/server.jar +Restart=always + +[Install] +WantedBy=multi-user.target diff --git a/shib_secondary/remote/check-and-toggle-mode.sh b/shib_secondary/remote/check-and-toggle-mode.sh new file mode 100755 index 0000000..bd35adc --- /dev/null +++ b/shib_secondary/remote/check-and-toggle-mode.sh @@ -0,0 +1,92 @@ +#!/bin/bash + +use_dnbd3() { + mountpoint -q /mnt/store && [ -x "/opt/dnbd3/dnbd3-server" ] +} + +is_dnbd3() { + systemctl -q is-active dnbd3-server.service || return 1 + systemctl -q is-active dnbd3-ssl-in.service || return 1 + systemctl -q is-active dnbd3-ssl-out.service || return 1 + return 0 +} + +enable_dnbd3() { + echo "Enabling dnbd3 proxy" + systemctl disable --now bwlp-master-socat@5006.service + systemctl enable --now dnbd3-server.service + systemctl enable --now dnbd3-ssl-in.service + systemctl enable --now dnbd3-ssl-out.service +} + +disable_dnbd3() { + echo "Disabling dnbd3 proxy" + systemctl disable --now dnbd3-server.service + systemctl disable --now dnbd3-ssl-in.service + systemctl disable --now dnbd3-ssl-out.service +} + +is_passthrough() { + systemctl -q is-active apache2.service || return 1 + systemctl -q is-active bwlp-master-socat@9090.service || return 1 + systemctl -q is-active bwlp-master-socat@9091.service || return 1 + local lnk=$( readlink -f /etc/apache2/sites-enabled/110-bwlp.conf ) + [ "$lnk" = "/etc/apache2/sites-available/110-bwlp-passthrough.conf" ] || return 1 + return 0 +} + +is_active() { + systemctl -q is-active apache2.service || return 1 + systemctl -q is-active bwlp-master.service || return 1 + local lnk=$( readlink -f /etc/apache2/sites-enabled/110-bwlp.conf ) + [ "$lnk" = "/etc/apache2/sites-available/110-bwlp-active.conf" ] || return 1 + return 0 +} + +passthrough() { + is_passthrough && return 0 + echo "Enabling passthrough" + systemctl disable --now bwlp-master.service + ln -nfs ../sites-available/110-bwlp-passthrough.conf /etc/apache2/sites-enabled/110-bwlp.conf + systemctl restart apache2.service + systemctl enable --now bwlp-master-socat@9090.service + systemctl enable --now bwlp-master-socat@9091.service + systemctl enable --now bwlp-master-socat@9050.service + systemctl enable --now bwlp-master-socat@9051.service + if ! use_dnbd3; then + systemctl enable --now bwlp-master-socat@5006.service + fi +} + +activate() { + is_active && return 0 + echo "Enabling active mode" + systemctl disable --now bwlp-master-socat@9090.service + systemctl disable --now bwlp-master-socat@9091.service + systemctl disable --now bwlp-master-socat@9050.service + systemctl disable --now bwlp-master-socat@9051.service + systemctl disable --now bwlp-master-socat@5006.service + ln -nfs ../sites-available/110-bwlp-active.conf /etc/apache2/sites-enabled/110-bwlp.conf + systemctl restart apache2.service + systemctl enable --now bwlp-master.service +} + +# Check connectivity + +if curl -L -m 10 -o /dev/null -sS \ + --retry-max-time 40 --retry 4 --retry-all-errors \ + https://%DOMAIN%/webif/; then + # OK + passthrough +else + # Take over + activate +fi + +if use_dnbd3; then + is_dnbd3 || enable_dnbd3 +else + is_dnbd3 && disable_dnbd3 +fi + +exit 0 diff --git a/shib_secondary/remote/config.php b/shib_secondary/remote/config.php new file mode 100644 index 0000000..52f6460 --- /dev/null +++ b/shib_secondary/remote/config.php @@ -0,0 +1,36 @@ + 0 )); do + case "$1" in + --domain) + domain="$2" + shift + ;; + --master) + master="$2" + shift + ;; + --secret) + secret="$2" + shift + ;; + *) + echo "WAAT? NO MAHNEEY?" + exit 1 + esac + shift +done + +cd "$(dirname "$( readlink -f "$0" )" )" || cd /tmp/shib_deploy || exit 1 + +# Replace domain in everything +find . -type f -exec sed -i "s/%DOMAIN%/$domain/g" {} \; + +dest="/opt/bwlp" +mkdir -p "$dest" +mkdir -p "/etc/dnbd3-server" +cp check-and-toggle-mode.sh "$dest/" +cp copy-current-config.sh "$dest/" +cp bwlp-*.{service,timer} "/etc/systemd/system/" +cp dnbd3-*.service "/etc/systemd/system/" +cp ??0-bwlp-*.conf "/etc/apache2/sites-available/" +cp server.conf alt-servers "/etc/dnbd3-server/" + +sed -i "/$domain/d" "/etc/hosts" +echo "$master $domain" >> "/etc/hosts" + +apt install -y apache2 socat libjansson4 \ + libapache2-mod-php php-curl php-json php-mbstring php-mysql \ + libapache2-mod-shib mariadb-server default-jre-headless + +a2enmod proxy proxy_http ssl headers +mkdir -p /var/log/apache2/masterserver +rm -f -- /etc/apache2/sites-enabled/*.conf +ln -nfs ../sites-available/000-bwlp-default.conf /etc/apache2/sites-enabled/000-bwlp-default.conf + +if ! id bwlp; then + adduser --disabled-password --comment 'bwlp-user' bwlp +fi + +echo "SHARED_SECRET='$secret'" > /opt/bwlp/config + +mariadb <