[mount-store] Beefed up retry logic

1 files changed, 71 insertions, 48 deletions

diff --git a/scripts/mount-store b/scripts/mount-store
index 260b0c7..cb6b769 100755
--- a/scripts/mount-store
+++ b/scripts/mount-store
@@ -26,6 +26,63 @@ esac
 FLAG="${DEST}/.notmounted"
 SUBDIR="${DEST}/bwlehrpool_store"
 
+storage_test () {
+	rm -f -- "${FLAG}"
+	if [ -e "${FLAG}" ]; then
+		echo "Error: File '.notmounted' exists on remote storage and could not be deleted." >&2
+		echo "Error: Make sure the share is writable." >&2
+		return 5
+	fi
+	chgrp images "${DEST}" 2>/dev/null
+	mkdir -p "${SUBDIR}"
+	if [ ! -d "${SUBDIR}" ]; then
+		echo "Error: Could not create $(basename "$SUBDIR")! Storage not writable!" >&2
+		return 6
+	fi
+	echo "Applying group..."
+	find "${SUBDIR}" -type d -exec chgrp images {} \; 2>/dev/null
+	echo "Applying permissions..."
+	find "${SUBDIR}" -type d -exec chmod ug+rwx {} \; 2>/dev/null
+	echo "Creating test file..."
+	local TEST="${SUBDIR}/.deleteme-$RANDOM-$RANDOM"
+	sudo -n -u dmsd touch "$TEST"
+	local RET=$?
+	if [ -e "$TEST" ]; then
+		sudo -n -u dmsd rm -f -- "$TEST"
+	else
+		[ "$RET" = "0" ] && RET=127
+		echo "Error: Mounted share is not writable." >&2
+		ls -al "${DEST}" "${SUBDIR}" >&2
+	fi
+	return $RET
+}
+
+# Already mounted?
+TRIES=0
+while awk '{print $2}' "/proc/mounts" | grep -q "^${DEST}\$"; do
+	echo "Trying to unmount '$DEST'..."
+	if [ "$TRIES" -gt 5 ]; then
+		echo "Error: Cannot unmount '$DEST', giving up." >&2
+		exit "$RET"
+	elif [ "$TRIES" -gt 3 ]; then
+		umount -v -f "$DEST"
+	else
+		umount -v "$DEST"
+	fi
+	RET=$?
+	if [ "$RET" -ne "0" ]; then
+		cat >&2 <<-HEREDOC
+			Error: Cannot unmount '$DEST'!
+			Storage might be in use (running VM upload etc.)
+			Try stopping DMSD first
+
+			If the problem persists, try rebooting the server.
+		HEREDOC
+		exit "$RET"
+	fi
+	TRIES=$(( $TRIES + 1 ))
+done
+
 # Sanity checks: Destination exists?
 if [ ! -d "$DEST" ]; then
 	mkdir -p "$DEST"
@@ -40,22 +97,9 @@ if [ ! -d "$DEST" ]; then
 	exit 1
 fi
 
-# Already mounted?
-if awk '{print $2}' "/proc/mounts" | grep -q "^${DEST}\$"; then
-	echo "Trying to unmount '$DEST'..."
-	umount -v "$DEST"
-	RET=$?
-	if [ "$RET" -ne "0" ]; then
-		echo "Cannot unmount '$DEST'!" >&2
-		echo "Storage might be in use (running VM upload etc.)" >&2
-		echo "Try stopping DMSD first" >&2
-		exit "$RET"
-	fi
-fi
-
 # Unmount and not requested to mount (local mode)
 if [[ "${SOURCE}" == "null" ]]; then
-	rm -f "${FLAG}"
+	rm -f -- "${FLAG}"
 	echo "Success. Now using internal storage."
 	exit 0
 fi
@@ -72,9 +116,15 @@ fi
 if grep -E -q '^[^/].+:.+' <<<$SOURCE; then
 	# seems to be NFS
 	for opt in vers=4 vers=3; do
+		echo " * Trying ${opt}..."
 		mount -v -t nfs -o rw,noatime,noexec,nodev,async,nolock,$opt,fg,ac,retry=1,timeo=150,sec=sys "$SOURCE" "$DEST"
 		RET=$?
+		[ "$RET" -ne "0" ] && continue
+		echo "Mount succeeded, checking write permissions...."
+		storage_test
+		RET=$?
 		[ "$RET" -eq "0" ] && break
+		umount -v "$DEST" || umount -v -f -l "$DEST"
 	done
 elif grep -E -q '^//' <<<$SOURCE; then
 	# seems to be SMB
@@ -84,10 +134,12 @@ elif grep -E -q '^//' <<<$SOURCE; then
 		echo " * Trying ${sec}..."
 		mount -v -t cifs -o rw,uid=0,gid=12345,forceuid,forcegid,nounix,file_mode=0664,dir_mode=0775,sec=$sec "$SOURCE" "$DEST"
 		RET=$?
-		if [ "$RET" -eq "0" ]; then
-			echo " * Success!"
-			break
-		fi
+		[ "$RET" -ne "0" ] && continue
+		echo "Mount succeeded, checking write permissions...."
+		storage_test
+		RET=$?
+		[ "$RET" -eq "0" ] && break
+		umount -v "$DEST" || umount -v -f -l "$DEST"
 	done
 	unset USER PASSWD
 else
@@ -95,36 +147,7 @@ else
 	exit 1
 fi
 
-echo "----------------------------------"
-
-if [ "$RET" == "0" ]; then
-	rm -f "${FLAG}"
-	if [ -e "${FLAG}" ]; then
-		echo "Error: File '.notmounted' exists on remote storage and could not be deleted." >&2
-		echo "Error: Make sure the share is writable." >&2
-		umount -v "$DEST"
-		exit 5
-	fi
-	chgrp images "${DEST}" 2>/dev/null
-	mkdir -p "${SUBDIR}"
-	if [ ! -d "${SUBDIR}" ]; then
-		echo "Error: Could not create $(basename "$SUBDIR")! Storage not writable!" >&2
-		umount -v "$DEST"
-		exit 6
-	fi
-	chgrp -R images "${SUBDIR}" 2>/dev/null &
-	chmod -R ug+rwX "${SUBDIR}" 2>/dev/null &
-	TEST="${SUBDIR}/.deleteme-$RANDOM-$RANDOM"
-	sudo -n -u dmsd touch "$TEST"
-	RET=$?
-	if [ -e "$TEST" ]; then
-		sudo -n -u dmsd rm -f -- "$TEST"
-	else
-		[ "$RET" = "0" ] && RET=127
-		echo "Error: Mounted share is not writable, aborting." >&2
-		umount -v "$DEST"
-	fi
-fi
+echo ""
 
 if [ "$RET" = "0" ]; then
 	echo "----------------------------------"