From a231e6283e1c79a33d6f3bdf786832f10e355d0c Mon Sep 17 00:00:00 2001
From: Simon Rettberg
Date: Fri, 17 Jul 2020 18:19:58 +0200
Subject: [KERNEL] Trying to get locking under control

Current code doesn't pay much attention to properly locking when
accessing structures that are being used in multiple workers/threads.
Try to fix this, slowly.
---
 src/kernel/mq.c | 55 ++++++++++++++++++++++++++++++-------------------------
 1 file changed, 30 insertions(+), 25 deletions(-)

(limited to 'src/kernel/mq.c')

diff --git a/src/kernel/mq.c b/src/kernel/mq.c
index f9e1bd2..cc710b2 100644
--- a/src/kernel/mq.c
+++ b/src/kernel/mq.c
@@ -89,14 +89,14 @@ static void dnbd3_busy_iter_requeue(struct request *req, void *priv, bool arg)
 		/* request is in sending, so we will not requeue */
 		return;
 	}
-	if (!cmd->dnbd3->connected) {
+	if (!cmd->dnbd3->socks) {
 		dnbd3_end_cmd(cmd, BLK_STS_IOERR);
 		mutex_unlock(&cmd->lock);
 		return;
 	}
 	debug_sock(sock, "requeue busy request %p", req);
-	dnbd3_requeue_cmd(cmd);
 	mutex_unlock(&cmd->lock);
+	dnbd3_requeue_cmd(cmd);
 }
 
 /**
@@ -116,16 +116,27 @@ void dndb3_reque_busy_requests(struct dnbd3_sock *sock)
 static void dnbd3_wait_for_sockets(struct dnbd3_device *dev) {
 	int i;
 	bool wait = true;
-	while (wait && dev->connected) {
+	bool msg = false;
+	mutex_lock(&dev->device_lock);
+	while (dev->socks) {
 		for (i = 0; i < dev->number_connections; i++) {
 			if (dnbd3_is_sock_alive(dev->socks[i])) {
 				wait = false;
 			}
 		}
-		if (wait && dev->connected) {
-			debug("no socket alive sleeping");
-			msleep(100);
+		if (!wait)
+			break;
+		mutex_unlock(&dev->device_lock);
+		if (!msg) {
+			debug_dev(dev, "no socket alive sleeping");
+			msg = true;
 		}
+		msleep(100);
+		mutex_lock(&dev->device_lock);
+	}
+	mutex_unlock(&dev->device_lock);
+	if (msg) {
+		debug_dev(dev, "done sleeping");
 	}
 }
 
@@ -136,7 +147,7 @@ static void dnbd3_wait_for_sockets(struct dnbd3_device *dev) {
 void dnbd3_requeue_cmd(struct dnbd3_cmd *cmd)
 {
 	struct request *req = blk_mq_rq_from_pdu(cmd);
-	dnbd3_wait_for_sockets(cmd->dnbd3);
+	//dnbd3_wait_for_sockets(cmd->dnbd3);
 
 	if (!cmd->requed/* && blk_queued_rq(req)*/) {
 		cmd->requed = true;
@@ -158,6 +169,7 @@ void dnbd3_end_cmd(struct dnbd3_cmd *cmd, blk_status_t error)
 
 /**
  * dnbd3_handle_cmd - handles a mq command
+ * !! cmd->lock is held when being called !!
  * @cmd: the cmd to send
  * @index: the index of the queue
  */
@@ -165,25 +177,19 @@ static int dnbd3_handle_cmd(struct dnbd3_cmd *cmd, int index)
 {
 	struct request *req = blk_mq_rq_from_pdu(cmd);
 	struct dnbd3_device *dev = cmd->dnbd3;
-	struct dnbd3_sock *sock = &dev->socks[index];
+	struct dnbd3_sock *sock;
 	int ret = -1;
 
 //	debug_dev(dev, "handle request at position %lu, size %d, index %d",
 //			blk_rq_pos(req), blk_rq_bytes(req), index);
 
 
-	if (!dev->socks || !dnbd3_is_sock_alive(*sock)) {
-
-		if (dev->connected) {
-			info_dev(dev, "reset request to new socket");
-			dnbd3_requeue_cmd(cmd);
-			return 0;
-		} else {
-			error_dev(dev, "ending request device not connected");
-			dnbd3_end_cmd(cmd, BLK_STS_IOERR);
-			return -EIO;
-		}
+	if (!dev->socks || !dnbd3_is_sock_alive(dev->socks[index])) {
+		error_dev(dev, "ending request device not connected");
+		dnbd3_end_cmd(cmd, BLK_STS_IOERR);
+		return -EIO;
 	}
+	sock = &dev->socks[index];
 
 	mutex_lock(&sock->tx_lock);
 	if (unlikely(!sock->sock)) {
@@ -270,16 +276,15 @@ static enum blk_eh_timer_return dnbd3_xmit_timeout(struct request *req,
 		return BLK_EH_RESET_TIMER;
 	}
 
-	if (dev->connected) {
+	if (dev->socks) {
 		info_dev(dev, "reset request to new socket");
+		mutex_unlock(&cmd->lock);
 		dnbd3_requeue_cmd(cmd);
+	} else {
+		error_dev(dev, "connection timed out");
+		dnbd3_end_cmd(cmd, BLK_STS_TIMEOUT);
 		mutex_unlock(&cmd->lock);
-		return BLK_EH_DONE;
 	}
-
-	error_dev(dev, "connection timed out");
-	dnbd3_end_cmd(cmd, BLK_STS_TIMEOUT);
-	mutex_unlock(&cmd->lock);
 	return BLK_EH_DONE;
 }
 
-- 
cgit v1.2.3-55-g7522