1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
|
#!/bin/ash
# This scripts runs as root and prepares the configuration
# for the dnbd3 server (...)
ERRLOG="/run/dnbd3-proxy.err"
[ -s "$ERRLOG" ] && exit 0 # already ran and failed, don't do it again
. /opt/openslx/bin/slx-tools
. /opt/openslx/config
# Normally, exit unclean to signal to systemd
islocal=
mingb=10
if [ "$SLX_RUNMODE_MODULE" != "dnbd3" ]; then
# But if its local proxy mode, always exit 0 since we don't consider this an error
islocal=true
# Not a dnbd3 proxy, see if we have a large partition to try local caching
if [ "$SLX_DNBD3_SERVERS" = "127.0.0.1" ]; then
# Already been here, do nothing
exit 0
fi
[ "x$SLX_VM_DNBD3" != "xyes" ] && exit 0 # Do not want
[ -z "$SLX_DNBD3_SERVERS" ] && exit 0 # No servers!?
[ -z "$SLX_DNBD3_MIN_GB" ] && exit 0 # Disabled
[ "$SLX_DNBD3_MIN_GB" -ge "$mingb" ] || exit 0 # Disabled
mingb="$SLX_DNBD3_MIN_GB"
fi
# This should never exist at bootup, but to be sure, try to remove it
# so we don't assume local caching in the dnbd3-server startup script
rmdir /mnt/dnbd3 2> /dev/null
# Log error to server and local file in /run.
# The main startup script will check whether this file is > 0 bytes
# and start in errormsg-mode.
errormsg () {
echo "$@" >> "$ERRLOG"
if [ -z "$islocal" ]; then
slxlog -s -e "dnbd3-setup" "$@"
exit 1
fi
exit 0
}
# Get size of disk/device at given path, in GB
disksize () {
local val len
val="$( fs_path_space "$1" )"
val="${val#* }"
len="${#val}"
if [ "$len" -lt 7 ]; then
echo 0
else
echo "${val:0:$(( len - 6 ))}"
fi
}
DNBD3_PORT=5003
# Creates the DNBD3 server configuration under DNBD3_CONF_DIR
DNBD3_CONF_DIR="/etc/dnbd3-server"
if ! mkdir -p "${DNBD3_CONF_DIR}"; then
errormsg "Failed to create '${DNBD3_CONF_DIR}'."
fi
# We prefer ID45 for storage (if >= 10GB), otherwise
# fallback to /tmp if it's not in RAM and >= 10GB.
# For local caching, /tmp is never used.
d="/opt/openslx/persistent"
dsd="$(disksize "$d")"
e="/tmp"
dse="$(disksize "$e")"
if fs_path_ismountpoint "$d" && [ -k "$d/data" ] && [ "$dsd" -ge "$mingb" ]; then
DNBD3_BASE_DIR="$d/data"
disksize="$dsd"
elif [ -z "$islocal" ] && ! fs_path_isvolatile "$e" && [ "$dse" -ge "$mingb" ]; then
DNBD3_BASE_DIR="$e"
disksize="$dse"
# We're using /tmp, don't warn about low space
touch "/run/openslx/no-ram-warning"
else
errormsg "Neither a persistent part (ID45) nor /tmp (ID44) are available, or they are < ${mingb}GB"
fi
# now try to create the actual folder used by the server
DNBD3_DATA_DIR="${DNBD3_BASE_DIR}/dnbd3"
if ! mkdir -p "${DNBD3_DATA_DIR}"; then
for i in 0 1 2 3 4 5 $RANDOM $RANDOM; do
if mkdir -p "${DNBD3_BASE_DIR}/dnbd3.$i"; then
DNBD3_DATA_DIR="${DNBD3_BASE_DIR}/dnbd3.$i"
break
fi
done
fi
if [ ! -d "${DNBD3_DATA_DIR}" ]; then
errormsg "Failed to create '${DNBD3_BASE_DIR}/dnbd3' or any of the fallback variants"
fi
if ! chown -R dnbd3:dnbd3 "${DNBD3_DATA_DIR}"; then
errormsg "Failed to chown '${DNBD3_DATA_DIR}' to dnbd3."
fi
# user rw, group ro, other nothing
if ! chmod -R o-rwx,u+rwX,g+rX-w "${DNBD3_DATA_DIR}"; then
errormsg "Failed to chmod '${DNBD3_DATA_DIR}' to dnbd3."
fi
# Done with sanity checks, now create configs: server.conf & alt-servers
# Using the information given by the server in /opt/openslx/config
# helper to echo given list of IPs to ${DNBD3_CONF_DIR}/alt-servers
# optionally takes a single char prefix as first param and
# adds it to the IP (for private dnbd3 servers)
# Also sets FOUND_SAT if satellite IP was seen
MY_IPS=$(ip a | grep '^\s*inet\s' | awk '{print $2}')
FOUND_SAT=
add_alt_server() {
local PRE=
if [ "${#1}" = "1" ]; then
PRE="$1"
shift
fi
for ALT in "$@"; do
for ip in $MY_IPS; do
[ "x$ALT" = "x${ip%/*}" ] && return 0 # Ignore self
done
echo "${PRE}${ALT}" >> "${DNBD3_CONF_DIR}/alt-servers"
[ "x${ALT}" = "x${SLX_PXE_SERVER_IP}" ] && FOUND_SAT="oui"
done
return 0
}
rm -f -- "${DNBD3_CONF_DIR}/alt-servers"
# Local caching mode?
if [ -n "$islocal" ]; then
# Local caching mode.
DNBD3_SPARSE="true"
DNBD3_PRETEND_CLIENT="true"
DNBD3_AUTOFREE=0 # Immediately start freeing images when running out of space in local caching mode
if [ -n "$SLX_DNBD3_MIN_GB_HASH" ] && [ "$SLX_DNBD3_MIN_GB_HASH" -gt 0 ] && [ "$disksize" -ge "$SLX_DNBD3_MIN_GB_HASH" ]; then
DNBD3_BGR="hashblock"
else
DNBD3_BGR="false"
fi
DNBD3_LOOKUP="false"
DNBD3_SERVER_PENALTY="500000"
DNBD3_BGR_MIN_CLIENTS=1
MAX_REPLICATION_SIZE=500
# Use DNBD3 servers from openslx config and then patch it to say localhost
add_alt_server '-' ${SLX_DNBD3_SERVERS}
sed -i "s/^SLX_DNBD3_SERVERS=.*$/SLX_DNBD3_SERVERS='127.0.0.1'/;s/^SLX_DNBD3_FALLBACK=.*$/SLX_DNBD3_FALLBACK='${SLX_DNBD3_SERVERS} ${SLX_DNBD3_FALLBACK}'/" '/opt/openslx/config'
else
# Proxy mode
DNBD3_SPARSE="true"
DNBD3_PRETEND_CLIENT="false"
DNBD3_BGR="false"
DNBD3_BGR_MIN_CLIENTS=0
DNBD3_LOOKUP="false"
# Delay deletion only if we're booted early
hour="$( date +%_H )"
hour="${hour# }"
if [ "$hour" -lt 7 ]; then
DNBD3_AUTOFREE=$(( 7 - hour ))
else
DNBD3_AUTOFREE=0
fi
# Wait some more on Sunday
if [ "$( date +%w )" = 0 ]; then
DNBD3_AUTOFREE=$(( DNBD3_AUTOFREE + 18 ))
fi
DNBD3_SERVER_PENALTY=3000 # no BGR = don't like other servers connecting so much
if [ -n "${SLX_DNBD3_BGR}" ]; then
DNBD3_BGR="true"
DNBD3_SERVER_PENALTY=500 # much better
# Only do chained lookup of image if we're a global proxy with BGR
[ -z "${SLX_DNBD3_WHITELIST}" ] && DNBD3_LOOKUP="true"
fi
# Determine if we need sparse mode, or override BGR
if ! [ "$disksize" -gt 200 ]; then
# less than 120GiB or NaN, force
[ "$DNBD3_BGR" = "true" ] && slxlog -s -e "dnbd3-setup-small" "Cache partition < 200GiB, forcing background replication OFF and sparse mode ON"
DNBD3_BGR="false"
DNBD3_SERVER_PENALTY=5000
DNBD3_LOOKUP="false"
elif [ "$disksize" -gt 2000 ]; then
# force non-sparse on > 2TiB
DNBD3_SPARSE="false"
fi
MAX_REPLICATION_SIZE=$(( disksize / 6 ))
[ "$MAX_REPLICATION_SIZE" -lt 100 ] && MAX_REPLICATION_SIZE=100
add_alt_server ${SLX_DNBD3_PUBLIC}
add_alt_server '-' ${SLX_DNBD3_PRIVATE}
rm -f "/opt/openslx/iptables/rules.d/99-dnbd3"
# now create iptables helper rules
if [ -n "${SLX_DNBD3_WHITELIST}" ]; then
DNBD3_IPTABLES_CONF="$(mktemp)"
echo '#!/bin/ash' > "${DNBD3_IPTABLES_CONF}"
for CIDR in ${SLX_DNBD3_WHITELIST} ${SLX_KCL_SERVERS}; do
echo "iptables -I ipt-helper-INPUT 1 -p tcp -s ${CIDR} --dport ${DNBD3_PORT} -j ACCEPT"
done >> "${DNBD3_IPTABLES_CONF}"
echo "iptables -A ipt-helper-INPUT -p tcp --dport ${DNBD3_PORT} -j REJECT" >> "${DNBD3_IPTABLES_CONF}"
chmod +x "${DNBD3_IPTABLES_CONF}"
mv -f "$DNBD3_IPTABLES_CONF" "/opt/openslx/iptables/rules.d/99-dnbd3"
fi
(
echo "*******************"
echo "*** DNBD3 Proxy ***"
echo "*******************"
) | tee -a "/etc/issue" -a "/etc/issue.net" >> "/opt/openslx/etc/issue.template"
# Create a crontab for rebooting - if everything is fine, once a weekend,
# on failure, reboot every night, hoping things will get better (...)
M=$(( RANDOM % 60 ))
H=$(( RANDOM % 5 ))
if [ -s "$ERRLOG" ]; then
W="*"
else
W="0"
fi
# Uptime >10hrs, or we might reboot multiple times if $H is > last $H
cat > "/etc/cron.d/dnbd3-reboot" <<EOF
# OpenSLX: Reboot proxy at night
SHELL=/bin/ash
PATH=/usr/sbin:/usr/bin:/sbin:/bin:/opt/openslx/sbin:/opt/openslx/bin
$M $H * * $W root [ "\$(grep -oEm1 '^[0-9]+' /proc/uptime)" -lt 36000 ] || reboot
EOF
#
fi # end local caching or proxy mode
rm -f "${DNBD3_CONF_DIR}/server.conf"
# Refer to http://git.openslx.org/dnbd3.git/tree/conf for configuration options
cat << EOF > "${DNBD3_CONF_DIR}/server.conf"
[dnbd3]
listenPort=${DNBD3_PORT}
basePath=${DNBD3_DATA_DIR}
serverPenalty=${DNBD3_SERVER_PENALTY}
clientPenalty=0
isProxy=true
backgroundReplication=${DNBD3_BGR}
bgrMinClients=${DNBD3_BGR_MIN_CLIENTS}
bgrWindowSize=1
lookupMissingForProxy=${DNBD3_LOOKUP}
sparseFiles=${DNBD3_SPARSE}
removeMissingImages=false
uplinkTimeout=5000
clientTimeout=15000
pretendClient=${DNBD3_PRETEND_CLIENT}
autoFreeDiskSpaceDelay=${DNBD3_AUTOFREE}h
[limits]
maxPayload=10M
maxReplicationSize=${MAX_REPLICATION_SIZE}G
[logging]
consoleMask=ERROR WARNING MINOR INFO
EOF
# Extra overrides - make this proerly ini aware some day. For now we don't
# have any name clashes between the ini sections so this works
for item in $SLX_DNBD3_EXTRA; do # space sep
item="${item#*.}"
key="${item%%=*}"
val="${item#*=}"
sed -i "s/^${key}=.*$/${key}=${val}/" "${DNBD3_CONF_DIR}/server.conf"
done
# To this day, only the sat IP is in SLX_KCL_SERVERS afaik
[ -z "${FOUND_SAT}" ] && add_alt_server ${SLX_KCL_SERVERS}
chmod -R a+Xr "${DNBD3_CONF_DIR}"
# create rpc.acl to allow the satellite only
rm -f "${DNBD3_CONF_DIR}/rpc.acl"
for SRV in ${SLX_KCL_SERVERS}; do
echo "$SRV ALL"
done >> "${DNBD3_CONF_DIR}/rpc.acl"
# And self
echo "127.0.0.1 STATS IMAGE_LIST" >> "${DNBD3_CONF_DIR}/rpc.acl"
if [ -n "$islocal" ]; then
# Prepare, so dnbd3-server will be launched with -m
mkdir -p /mnt/dnbd3
chown dnbd3:dnbd3 /mnt/dnbd3
# Start service, is not symlinked when not in proxy mode
systemctl --no-block start dnbd3-proxy.service dnbd3-local-stage4.service
fi
exit 0
|