From dc8fcb26807c734cf79d8a82d2ba655f226ad135 Mon Sep 17 00:00:00 2001
From: Frederic Robra
Date: Wed, 4 Sep 2019 15:10:03 +0200
Subject: prepared kernel module to have configurable number of connections

---
 src/kernel/core.c  | 42 +++++++++++++++++++++++------------------
 src/kernel/dnbd3.h | 11 ++---------
 src/kernel/net.c   | 55 +++++++++++++++++++++++++++++++++++++++++-------------
 src/kernel/sysfs.c |  2 +-
 4 files changed, 69 insertions(+), 41 deletions(-)

(limited to 'src/kernel')

diff --git a/src/kernel/core.c b/src/kernel/core.c
index 6aa3b50..2f7ddcb 100644
--- a/src/kernel/core.c
+++ b/src/kernel/core.c
@@ -74,6 +74,7 @@ static int dnbd3_ioctl(struct block_device *bdev, fmode_t mode,
 		unsigned int cmd, unsigned long arg)
 {
 	int result = -EIO;
+	bool old_msg = false;
 	struct dnbd3_device *dev = bdev->bd_disk->private_data;
 	char *imgname = NULL;
 	dnbd3_ioctl_t *msg = NULL;
@@ -87,7 +88,12 @@ static int dnbd3_ioctl(struct block_device *bdev, fmode_t mode,
 			return -ENOMEM;
 		}
 		result = copy_from_user((char *)msg, (char *)arg, 2);
-		if (result != 0	|| msg->len != sizeof(*msg)) {
+		if (result != 0) {
+			result = -ENOEXEC;
+			goto error;
+		} else if (msg->len == 40){ /* old msg length was 40 */
+			old_msg = true;
+		} else if (msg->len != sizeof(*msg)) {
 			result = -ENOEXEC;
 			goto error;
 		}
@@ -146,6 +152,11 @@ static int dnbd3_ioctl(struct block_device *bdev, fmode_t mode,
 			memcpy(dev->alt_servers, &dev->initial_server,
 					sizeof(dev->alt_servers[0]));
 
+			if (old_msg || msg->number_connections == 0) {
+				dev->number_connections = 4;
+			} else {
+				dev->number_connections = msg->number_connections;
+			}
 			mutex_unlock(&dev->device_lock);
 
 			result = dnbd3_net_connect(dev);
@@ -209,8 +220,12 @@ static int dnbd3_ioctl(struct block_device *bdev, fmode_t mode,
 		break;
 	}
 error:
-	if (msg) kfree(msg);
-	if (imgname) kfree(imgname);
+	if (msg) {
+		kfree(msg);
+	}
+	if (imgname) {
+		kfree(imgname);
+	}
 	return result;
 
 }
@@ -240,19 +255,11 @@ int dnbd3_add_device(struct dnbd3_device *dev, int minor)
 	struct gendisk *disk;
 	struct request_queue *q;
 	int err = -ENOMEM;
-	int i;
 	debug("adding device %d", minor);
 
 	mutex_init(&dev->device_lock);
 	mutex_lock(&dev->device_lock);
 
-//	atomic_set(&dev->mq_requests, 0);
-	for (i = 0; i < NUMBER_CONNECTIONS; i++) {
-		dev->socks[i].device = dev;
-		dev->socks[i].sock_nr = i;
-		mutex_init(&dev->socks[i].tx_lock);
-	}
-
 	disk = alloc_disk(1);
 	if (!disk) {
 		error_dev(dev, "allocating disc failed");
@@ -265,14 +272,15 @@ int dnbd3_add_device(struct dnbd3_device *dev, int minor)
 		err = -EEXIST;
 	}
 
-	if (err < 0)
+	if (err < 0) {
 		goto out_free_disk;
+	}
 
 	dev->minor = minor;
 	dev->disk = disk;
 	dev->tag_set.ops = &dnbd3_mq_ops;
 	/* this can be changed later with blk_mq_update_nr_hw_queues() */
-	dev->tag_set.nr_hw_queues = NUMBER_CONNECTIONS;
+	dev->tag_set.nr_hw_queues = 1;
 	dev->tag_set.queue_depth = 128;
 	dev->tag_set.numa_node = NUMA_NO_NODE;
 	dev->tag_set.cmd_size = sizeof(struct dnbd3_cmd);
@@ -404,9 +412,10 @@ static void dnbd3_dev_remove(struct dnbd3_device *dev)
 {
 	struct gendisk *disk = dev->disk;
 	struct request_queue *q;
-	int i;
 
-	dnbd3_net_disconnect(dev);
+	if (dev->connected) {
+		dnbd3_net_disconnect(dev);
+	}
 
 	if (disk) {
 		q = disk->queue;
@@ -421,9 +430,6 @@ static void dnbd3_dev_remove(struct dnbd3_device *dev)
 		dev->imgname = NULL;
 	}
 	mutex_destroy(&dev->device_lock);
-	for (i = 0; i < NUMBER_CONNECTIONS; i++) {
-		mutex_destroy(&dev->socks[i].tx_lock);
-	}
 }
 
 /**
diff --git a/src/kernel/dnbd3.h b/src/kernel/dnbd3.h
index 79484a9..6d8ceb2 100644
--- a/src/kernel/dnbd3.h
+++ b/src/kernel/dnbd3.h
@@ -30,14 +30,6 @@
 #include "types.h"
 #include "serialize.h"
 
-/**
- * the number of parallel connections
- */
-
-#define NUMBER_CONNECTIONS 4
-
-
-
 
 /**
  * factor to which the next server is choosen or the same
@@ -149,7 +141,8 @@ struct dnbd3_device {
 	struct mutex device_lock;
 
 	bool connected;
-	struct dnbd3_sock socks[NUMBER_CONNECTIONS];
+	uint8_t number_connections;
+	struct dnbd3_sock *socks;
 	char *imgname;
 	struct dnbd3_server initial_server;
 	struct dnbd3_server alt_servers[NUMBER_SERVERS];//TODO just add server to list do not remove
diff --git a/src/kernel/net.c b/src/kernel/net.c
index f4665bf..061deb1 100644
--- a/src/kernel/net.c
+++ b/src/kernel/net.c
@@ -73,7 +73,7 @@ static void dnbd3_timer(struct timer_list *arg)
 	busy = dnbd3_is_mq_busy(dev);
 
 	if (dev->timer_count % TIMER_INTERVAL_KEEPALIVE_PACKET == 0) {
-		for (i = 0; i < NUMBER_CONNECTIONS; i++) {
+		for (i = 0; i < dev->number_connections; i++) {
 			if (!test_bit(i, &busy) && dnbd3_is_sock_alive(dev->socks[i])) {
 				queue_work(dnbd3_wq, &dev->socks[i].keepalive_worker);
 			}
@@ -294,7 +294,7 @@ static int dnbd3_compare_plan(const void *lhs, const void *rhs)
 static void dnbd3_lock_all_socks(struct dnbd3_device *dev)
 {
 	int i;
-	for (i = 0; i < NUMBER_CONNECTIONS; i++) {
+	for (i = 0; i < dev->number_connections; i++) {
 		mutex_lock(&dev->socks[i].tx_lock);
 	}
 }
@@ -302,7 +302,7 @@ static void dnbd3_lock_all_socks(struct dnbd3_device *dev)
 static void dnbd3_unlock_all_socks(struct dnbd3_device *dev)
 {
 	int i;
-	for (i = 0; i < NUMBER_CONNECTIONS; i++) {
+	for (i = 0; i < dev->number_connections; i++) {
 		mutex_unlock(&dev->socks[i].tx_lock);
 	}
 }
@@ -312,7 +312,7 @@ static void dnbd3_print_conenction_plan(struct dnbd3_device *dev,
 {
 	int i;
 	debug_dev(dev, "connection plan:");
-	for (i = 0; i < NUMBER_CONNECTIONS; i++) {
+	for (i = 0; i < dev->number_connections; i++) {
 		debug_server(dev, plan[i], "server %d with avg rtt %llu:",
 				i, plan[i]->avg_rtt);
 	}
@@ -328,14 +328,20 @@ static void dnbd3_print_conenction_plan(struct dnbd3_device *dev,
  */
 static int dnbd3_adjust_connections(struct dnbd3_device *dev) {
 	int i, j, fallback, alive;
-	struct dnbd3_server *plan[NUMBER_CONNECTIONS];
+	struct dnbd3_server **plan;
 	struct dnbd3_server **servers = dnbd3_sort_server(dev);
 
+	plan = kmalloc(sizeof(struct dnbd3_server *) * dev->number_connections,
+			GFP_KERNEL);
+	if (!plan) {
+		error_dev(dev, "kmalloc failed");
+	}
+
 	if (servers && servers[0]->host.type != 0) {
 		plan[0] = servers[0];
 		fallback = 0;
 		j = 1;
-		for (i = 1; i < NUMBER_CONNECTIONS; i++) {
+		for (i = 1; i < dev->number_connections; i++) {
 			if (servers[j]->host.type != 0 &&
 					servers[j]->avg_rtt < RTT_UNKNOWN) {
 				if (RTT_FACTOR(plan[i - 1]->avg_rtt) >
@@ -353,13 +359,13 @@ static int dnbd3_adjust_connections(struct dnbd3_device *dev) {
 		}
 		kfree(servers);
 
-		sort(plan, NUMBER_CONNECTIONS, sizeof(struct dnbd3_server *),
+		sort(plan, dev->number_connections, sizeof(struct dnbd3_server *),
 			&dnbd3_compare_plan, NULL);
 		dnbd3_print_conenction_plan(dev, plan);
 
 		dnbd3_lock_all_socks(dev);
 		alive = 0;
-		for (i = 0; i < NUMBER_CONNECTIONS; i++) {
+		for (i = 0; i < dev->number_connections; i++) {
 			if (plan[i] != dev->socks[i].server ||
 					!dnbd3_is_sock_alive(dev->socks[i])) {
 
@@ -405,7 +411,7 @@ static void dnbd3_panic_worker(struct work_struct *work)
 	int i;
 	int sock_alive = 0;
 	dev = container_of(work, struct dnbd3_device, panic_worker);
-	for (i = 0; i < NUMBER_CONNECTIONS; i++) {
+	for (i = 0; i < dev->number_connections; i++) {
 		if (!dnbd3_is_sock_alive(dev->socks[i])
 				|| dev->socks[i].server->failures > 1000) {
 			panic = true;
@@ -445,7 +451,7 @@ static int dnbd3_meassure_rtt(struct dnbd3_device *dev,
 	int result;
 	uint64_t rtt = RTT_UNREACHABLE;
 	struct dnbd3_sock sock = {
-			.sock_nr = NUMBER_CONNECTIONS,
+			.sock_nr = dev->number_connections,
 			.sock = NULL,
 			.device = dev,
 			.server = server
@@ -505,11 +511,13 @@ static int dnbd3_meassure_rtt(struct dnbd3_device *dev,
 	rtt = (uint64_t)((end.tv_sec - start.tv_sec) * 1000000ull
 			+ (end.tv_usec - start.tv_usec));
 
-	debug_sock(&sock, "new rrt is %llu", rtt);
 
 error:
 	sock.server->rtts[dev->discovery_count % 4] = rtt;
 	sock.server->avg_rtt = dnbd3_average_rtt(sock.server);
+
+	debug_sock(&sock, "meassured rrt: %llu; avg_rtt: %llu", rtt,
+			sock.server->avg_rtt);
 	if (result <= 0) {
 		server->failures++;
 	}
@@ -846,8 +854,22 @@ static int dnbd3_socket_disconnect(struct dnbd3_sock *sock)
  */
 int dnbd3_net_connect(struct dnbd3_device *dev)
 {
-	int result;
+	int i, result;
 	debug_dev(dev, "connecting to server");
+	dev->socks = kmalloc(sizeof(struct dnbd3_sock) *
+			dev->number_connections, GFP_KERNEL);
+	if (!dev->socks) {
+		error_dev(dev, "kmalloc failed");
+	}
+	memset(dev->socks, 0, sizeof(struct dnbd3_sock) *
+			dev->number_connections);
+	for (i = 0; i < dev->number_connections; i++) {
+		dev->socks[i].device = dev;
+		dev->socks[i].sock_nr = i;
+		mutex_init(&dev->socks[i].tx_lock);
+	}
+	debug_dev(dev, "set nr hw queues to %d", dev->number_connections);
+	blk_mq_update_nr_hw_queues(&dev->tag_set, dev->number_connections);
 
 	if (dev->alt_servers[0].host.type == 0) {
 		return -ENONET;
@@ -888,19 +910,26 @@ int dnbd3_net_disconnect(struct dnbd3_device *dev)
 	int i;
 	int result = 0;
 
+	if (!dev->socks) {
+		info_dev(dev, "not connected");
+	}
+
 	del_timer_sync(&dev->timer);
 	/* be sure it does not recover while disconnecting */
 	cancel_work_sync(&dev->discovery_worker);
 	cancel_work_sync(&dev->panic_worker);
-	for (i = 0; i < NUMBER_CONNECTIONS; i++) {
+	for (i = 0; i < dev->number_connections; i++) {
 		if (dev->socks[i].sock) {
 			mutex_lock(&dev->socks[i].tx_lock);
 			if (dnbd3_socket_disconnect(&dev->socks[i])) {
 				result = -EIO;
 			}
 			mutex_unlock(&dev->socks[i].tx_lock);
+			mutex_destroy(&dev->socks[i].tx_lock);
 		}
 	}
+	kfree(dev->socks);
+	dev->socks = NULL;
 	dev->connected = false;
 	return result;
 }
diff --git a/src/kernel/sysfs.c b/src/kernel/sysfs.c
index 490ffbd..bd23845 100644
--- a/src/kernel/sysfs.c
+++ b/src/kernel/sysfs.c
@@ -42,7 +42,7 @@ ssize_t show_initial_server_addr(char *buf, struct dnbd3_device *dev)
 ssize_t show_connected_servers(char *buf, struct dnbd3_device *dev)
 {
 	int i, size = PAGE_SIZE, ret;
-	for (i = 0; i < NUMBER_CONNECTIONS; ++i) {
+	for (i = 0; i < dev->number_connections; ++i) {
 		if (dev->socks[i].sock) {
 			if (dev->socks[i].server->host.type == HOST_IP4) {
 				ret = MIN(snprintf(buf, size, "%pI4,%d,%llu,%d\n",
-- 
cgit v1.2.3-55-g7522