fixed some errors with multi queue

4 files changed, 54 insertions, 275 deletions

diff --git a/src/kernel/block.c b/src/kernel/block.c
index 26caa25..ade02f0 100644
--- a/src/kernel/block.c
+++ b/src/kernel/block.c
@@ -15,22 +15,6 @@
 
 
 
-static int dnbd3_open(struct block_device *bdev, fmode_t mode)
-{
-	dnbd3_device_t *dev = bdev->bd_disk->private_data;
-	printk(KERN_DEBUG "dnbd3: open device %i\n", dev->minor);
-	//TODO can be removed?
-	return 0;
-}
-
-static void dnbd3_release(struct gendisk *disk, fmode_t mode)
-{
-	dnbd3_device_t *dev = disk->private_data;
-	printk(KERN_DEBUG "dnbd3: release device %i\n", dev->minor);
-	//TODO can be removed?
-
-}
-
 
 void dnbd3_blk_fail_all_requests(dnbd3_device_t *dev)
 {
@@ -51,7 +35,7 @@ int dnbd3_net_connect(dnbd3_device_t *dev)
 		while (dev->disconnecting)
 			schedule();
 	}
-	if (dev->request) {
+	if (dev->pending) {
 		printk(KERN_DEBUG "dnbd3: device still in request\n");
 		while (dev->disconnecting)
 			schedule();
@@ -217,7 +201,7 @@ int dnbd3_net_connect(dnbd3_device_t *dev)
 	dev->panic_count = 0;
 
 	refcount_inc(&dev->config_refs);
-
+	mutex_init(&dev->socket_lock);
 	// Enqueue request to request_queue_send for a fresh list of alt servers
 	//TODO refresh alt server list
 
@@ -273,6 +257,26 @@ int dnbd3_net_disconnect(dnbd3_device_t *dev)
 	return 0;
 }
 
+static void printHost(struct dnbd3_host_t *host, char *msg)
+{
+	if (host->type == HOST_IP4) {
+		printk(KERN_INFO "dnbd3: %s %pI4:%d", msg, host->addr, host->port);
+	} else {
+		printk(KERN_INFO "dnbd3: %s [%pI6]:%d", msg, host->addr, host->port);
+	}
+}
+
+static void printServerList(struct dnbd3_device_t *dev)
+{
+	int i;
+	printHost(&dev->cur_server.host, "current server is");
+	for (i = 0; i < NUMBER_SERVERS; i++) {
+		if (dev->alt_servers[i].host.addr[0] != 0) {
+			printHost(&dev->alt_servers[i].host, "alternative server is");
+		}
+	}
+}
+
 static int dnbd3_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd, unsigned long arg)
 {
 	int result = -100;
@@ -343,6 +347,9 @@ static int dnbd3_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd
 //#else
 //			blk_queue->backing_dev_info.ra_pages = (msg->read_ahead_kb * 1024) / PAGE_SIZE;
 //#endif
+
+			printServerList(dev);
+
 			if (dnbd3_net_connect(dev) == 0) {
 				result = 0;
 				imgname = NULL; // Prevent kfree at the end
@@ -410,8 +417,6 @@ cleanup_return:
 struct block_device_operations dnbd3_fops =
 {
 	.owner = THIS_MODULE,
-	.open = dnbd3_open,
-	.release = dnbd3_release,
 	.ioctl = dnbd3_ioctl,
 	.compat_ioctl =	dnbd3_ioctl,
 };
diff --git a/src/kernel/core.c b/src/kernel/core.c
index 731c094..e92bd9a 100644
--- a/src/kernel/core.c
+++ b/src/kernel/core.c
@@ -61,226 +61,6 @@ static dnbd3_device_t *dnbd3_device;
 int major;
 
 
-//static int dnbd3_open(struct block_device *bdev, fmode_t mode)
-//{
-//	dnbd3_device_t *dev = bdev->bd_disk->private_data;
-//	printk(KERN_DEBUG "dnbd3: open device %i", dev->minor);
-//
-//	return 0;
-//}
-//
-//static void dnbd3_release(struct gendisk *disk, fmode_t mode)
-//{
-//	dnbd3_device_t *dev = disk->private_data;
-//	printk(KERN_DEBUG "dnbd3: release device %i", dev->minor);
-//
-//}
-//
-//
-//void dnbd3_blk_fail_all_requests(dnbd3_device_t *dev)
-//{
-//	printk(KERN_DEBUG "dnbd3: fail all requests device %i", dev->minor);
-//}
-//
-//
-//int dnbd3_net_connect(dnbd3_device_t *dev)
-//{
-//	printk(KERN_DEBUG "dnbd3: net connect device %i", dev->minor);
-//	return 0;
-//}
-//
-//
-//int dnbd3_net_disconnect(dnbd3_device_t *dev)
-//{
-//	printk(KERN_DEBUG "dnbd3: net disconnect device %i", dev->minor);
-//	return 0;
-//}
-//
-//static int dnbd3_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd, unsigned long arg)
-//{
-//	int result = -100;
-//	dnbd3_device_t *dev = bdev->bd_disk->private_data;
-//	char *imgname = NULL;
-//	dnbd3_ioctl_t *msg = NULL;
-//
-//	printk(KERN_DEBUG "dnbd3: ioctl device %i, cmd %i, arg %lu", dev->minor, cmd, arg);
-//	//unsigned long irqflags;
-//
-//	while (dev->disconnecting) {
-//		// do nothing
-//	}
-//
-//	if (arg != 0) {
-//		msg = kmalloc(sizeof(*msg), GFP_KERNEL);
-//		if (msg == NULL) return -ENOMEM;
-//		if (copy_from_user((char *)msg, (char *)arg, 2) != 0 || msg->len != sizeof(*msg)) {
-//			result = -ENOEXEC;
-//			goto cleanup_return;
-//		}
-//		if (copy_from_user((char *)msg, (char *)arg, sizeof(*msg)) != 0) {
-//			result = -ENOENT;
-//			goto cleanup_return;
-//		}
-//		if (msg->imgname != NULL && msg->imgnamelen > 0) {
-//			imgname = kmalloc(msg->imgnamelen + 1, GFP_KERNEL);
-//			if (imgname == NULL) {
-//				result = -ENOMEM;
-//				goto cleanup_return;
-//			}
-//			if (copy_from_user(imgname, msg->imgname, msg->imgnamelen) != 0) {
-//				result = -ENOENT;
-//				goto cleanup_return;
-//			}
-//			imgname[msg->imgnamelen] = '\0';
-//
-//			printk(KERN_DEBUG "dnbd3: ioctl image name of len %i is %s\n", (int)msg->imgnamelen, imgname);
-//		}
-//	}
-//
-//	switch (cmd) {
-//	case IOCTL_OPEN:
-//		printk(KERN_DEBUG "dnbd3: ioctl open");
-//		if (dev->imgname != NULL) {
-//			result = -EBUSY;
-//		} else if (imgname == NULL) {
-//			result = -EINVAL;
-//		} else if (msg == NULL) {
-//			result = -EINVAL;
-//		} else {
-//			if (sizeof(msg->host) != sizeof(dev->cur_server.host)) {
-//				printk(KERN_INFO "dnbd3: odd size bug#1 triggered in ioctl");
-//			}
-//			memcpy(&dev->cur_server.host, &msg->host, sizeof(msg->host));
-//			dev->cur_server.failures = 0;
-//			memcpy(&dev->initial_server, &dev->cur_server, sizeof(dev->initial_server));
-//			dev->imgname = imgname;
-//			dev->rid = msg->rid;
-//			dev->use_server_provided_alts = msg->use_server_provided_alts;
-//			// Forget all alt servers on explicit connect, set first al server to initial server
-//			memset(dev->alt_servers, 0, sizeof(dev->alt_servers[0])*NUMBER_SERVERS);
-//			memcpy(dev->alt_servers, &dev->initial_server, sizeof(dev->alt_servers[0]));
-////#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)
-////			if (blk_queue->backing_dev_info != NULL) {
-////				blk_queue->backing_dev_info->ra_pages = (msg->read_ahead_kb * 1024) / PAGE_SIZE;
-////			}
-////#else
-////			blk_queue->backing_dev_info.ra_pages = (msg->read_ahead_kb * 1024) / PAGE_SIZE;
-////#endif
-//			if (dnbd3_net_connect(dev) == 0) {
-//				result = 0;
-//				imgname = NULL; // Prevent kfree at the end
-//			} else {
-//				result = -ENOENT;
-//				dev->imgname = NULL;
-//			}
-//		}
-//		break;
-//
-//	case IOCTL_CLOSE:
-//		printk(KERN_DEBUG "dnbd3: ioctl close");
-//		dnbd3_blk_fail_all_requests(dev);
-//		result = dnbd3_net_disconnect(dev);
-//		dnbd3_blk_fail_all_requests(dev);
-//		set_capacity(dev->disk, 0);
-//		if (dev->imgname) {
-//			kfree(dev->imgname);
-//			dev->imgname = NULL;
-//		}
-//		break;
-//
-//	case IOCTL_SWITCH:
-//		printk(KERN_DEBUG "dnbd3: ioctl switch");
-//		result = -EINVAL;
-//		break;
-//
-//	case IOCTL_ADD_SRV:
-//	case IOCTL_REM_SRV:
-//		printk(KERN_DEBUG "dnbd3: ioctl add/rem srv");
-//		if (dev->imgname == NULL) {
-//			result = -ENOENT;
-//		} else if (dev->new_servers_num >= NUMBER_SERVERS) {
-//			result = -EAGAIN;
-//		} else if (msg == NULL) {
-//			result = -EINVAL;
-//		} else {
-//			memcpy(&dev->new_servers[dev->new_servers_num].host, &msg->host, sizeof(msg->host));
-//			dev->new_servers[dev->new_servers_num].failures = (cmd == IOCTL_ADD_SRV ? 0 : 1); // 0 = ADD, 1 = REM
-//			++dev->new_servers_num;
-//			result = 0;
-//		}
-//		break;
-//
-//	case BLKFLSBUF:
-//		printk(KERN_DEBUG "dnbd3: ioctl blkflsbuf");
-//		result = 0;
-//		break;
-//
-//	default:
-//		printk(KERN_DEBUG "dnbd3: ioctl unhandled cmd");
-//		result = -EIO;
-//		break;
-//	}
-//
-//cleanup_return:
-//	if (msg) kfree(msg);
-//	if (imgname) kfree(imgname);
-//	return result;
-//
-//}
-//
-//static const struct block_device_operations dnbd3_fops =
-//{
-//	.owner = THIS_MODULE,
-//	.open = dnbd3_open,
-//	.release = dnbd3_release,
-//	.ioctl = dnbd3_ioctl,
-//	.compat_ioctl =	dnbd3_ioctl,
-//};
-
-//static blk_status_t dnbd3_queue_rq(struct blk_mq_hw_ctx *hctx, const struct blk_mq_queue_data *bd)
-//{
-//	struct dnbd3_cmd *cmd = blk_mq_rq_to_pdu(bd->rq);
-//	int ret;
-//	struct dnbd3_device_t *dev;
-//
-//	printk(KERN_DEBUG "dnbd3: queue request device %i\n", dev->minor);
-//
-//	mutex_lock(&cmd->lock);
-//	clear_bit(1, &cmd->flags);
-//
-//
-//
-//	return 0;
-//}
-//
-//static void dnbd3_complete_rq(struct request *req)
-//{
-//	printk(KERN_DEBUG "dnbd3: dnbd3_complete_rq\n");
-//
-//}
-//
-//static int dnbd3_init_request(struct blk_mq_tag_set *set, struct request *rq, unsigned int hctx_idx, unsigned int numa_node)
-//{
-//	struct dnbd3_cmd *cmd = blk_mq_rq_to_pdu(rq);
-//	cmd->dnbd3 = set->driver_data;
-//	cmd->flags = 0;
-//	mutex_init(&cmd->lock);
-//	return 0;
-//}
-//static enum blk_eh_timer_return dnbd3_xmit_timeout(struct request *req, bool reserved)
-//{
-//	printk(KERN_DEBUG "dnbd3: dnbd3_xmit_timeout\n");
-//	return BLK_EH_DONE;
-//}
-//
-//
-//static const struct blk_mq_ops dnbd3_mq_ops = {
-//	.queue_rq = dnbd3_queue_rq,
-//	.complete = dnbd3_complete_rq,
-//	.init_request = dnbd3_init_request,
-//	.timeout = dnbd3_xmit_timeout,
-//};
-
 
 static int dnbd3_add_device(dnbd3_device_t *dev, int minor)
 {
@@ -293,7 +73,7 @@ static int dnbd3_add_device(dnbd3_device_t *dev, int minor)
 	disk = alloc_disk(1);
 	if (!disk) {
 		printk(KERN_WARNING "dnbd3: alloc_disc failed device %i\n", minor);
-		goto out_free_nbd;
+		goto out_free_dnbd3;
 	}
 
 	err = idr_alloc(&dnbd3_index_idr, dev, minor, minor + 1, GFP_KERNEL);
@@ -308,7 +88,7 @@ static int dnbd3_add_device(dnbd3_device_t *dev, int minor)
 	dev->minor = minor;
 	dev->disk = disk;
 	dev->tag_set.ops = &dnbd3_mq_ops;
-	dev->tag_set.nr_hw_queues = 1;
+	dev->tag_set.nr_hw_queues = 1; // this can be changed later with blk_mq_update_nr_hw_queues()
 	dev->tag_set.queue_depth = 128;
 	dev->tag_set.numa_node = NUMA_NO_NODE;
 	dev->tag_set.cmd_size = sizeof(dnbd3_cmd);
@@ -361,7 +141,7 @@ out_free_idr:
 	idr_remove(&dnbd3_index_idr, minor);
 out_free_disk:
 	put_disk(disk);
-out_free_nbd:
+out_free_dnbd3:
 	kfree(dev);
 	printk(KERN_DEBUG "dnbd3: destroy device %i\n", minor);
 	return err;
diff --git a/src/kernel/dnbd3.h b/src/kernel/dnbd3.h
index af9bdff..0a4ef67 100644
--- a/src/kernel/dnbd3.h
+++ b/src/kernel/dnbd3.h
@@ -44,7 +44,7 @@ typedef struct
 typedef struct dnbd3_device_t {
 	int minor;
 	struct blk_mq_tag_set tag_set;
-	struct request_queue queue;
+//	struct request_queue queue;
 	struct mutex config_lock;
 	refcount_t config_refs;
 	refcount_t refs;
@@ -67,7 +67,7 @@ typedef struct dnbd3_device_t {
 	dnbd3_server_t alt_servers[NUMBER_SERVERS]; // array of alt servers
 	int new_servers_num;	// number of new alt servers that are waiting to be copied to above array
 	dnbd3_server_entry_t new_servers[NUMBER_SERVERS]; // pending new alt servers
-	uint8_t discover, panic, disconnecting, update_available, panic_count, request;
+	uint8_t discover, panic, disconnecting, update_available, panic_count;
 	uint8_t use_server_provided_alts;
 	uint16_t rid;
 	uint32_t heartbeat_count;
@@ -99,15 +99,12 @@ typedef struct dnbd3_cmd {
 	unsigned long flags;
 	uint32_t cmd_cookie;
 } dnbd3_cmd;
-//
-//typedef struct dnbd3_sock {
-//	struct socket *sock;
-//	struct mutex tx_lock;
-//	struct request *pending;
-//	int sent;
-//	bool dead;
-//	int fallback_index;
-//	int cookie;
-//} dnbd3_sock;
+
+typedef struct dnbd3_sock {
+	struct socket *sock;
+	struct mutex lock;
+	struct request *pending;
+	struct dnbd3_server_t *server;
+} dnbd3_sock;
 
 #endif /* DNBD_H_ */
diff --git a/src/kernel/mq.c b/src/kernel/mq.c
index e3407ab..3b0bde5 100644
--- a/src/kernel/mq.c
+++ b/src/kernel/mq.c
@@ -11,7 +11,7 @@
 #include "block.h"
 
 #define DNBD3_CMD_REQUEUED 1
-
+//
 //static void dnbd3_config_put(struct dnbd3_device_t *dev)
 //{
 //	if (refcount_dec_and_mutex_lock(&dev->config_refs,
@@ -44,21 +44,21 @@ static int dnbd3_send_cmd(struct dnbd3_device_t *dev, struct dnbd3_cmd *cmd, int
 	sigset_t blocked, oldset;
 	void *kaddr;
 	int result;
-
+	dev->pending = req;
 	init_msghdr(msg);
 
 	dnbd3_request.magic = dnbd3_packet_magic;
 
 	switch (req_op(req)) {
-	case REQ_OP_DISCARD:
-		printk(KERN_DEBUG "dnbd3: request operation discard on device %d\n", dev->minor);
-		break;
-	case REQ_OP_FLUSH:
-		printk(KERN_DEBUG "dnbd3: request operation flush on device %d\n", dev->minor);
-		break;
-	case REQ_OP_WRITE:
-		printk(KERN_DEBUG "dnbd3: request operation write on device %d\n", dev->minor);
-		break;
+//	case REQ_OP_DISCARD:
+//		printk(KERN_DEBUG "dnbd3: request operation discard on device %d\n", dev->minor);
+//		break;
+//	case REQ_OP_FLUSH:
+//		printk(KERN_DEBUG "dnbd3: request operation flush on device %d\n", dev->minor);
+//		break;
+//	case REQ_OP_WRITE:
+//		printk(KERN_DEBUG "dnbd3: request operation write on device %d\n", dev->minor);
+//		break;
 	case REQ_OP_READ:
 		printk(KERN_DEBUG "dnbd3: request operation read on device %d\n", dev->minor);
 		dnbd3_request.cmd = CMD_GET_BLOCK;
@@ -81,7 +81,7 @@ static int dnbd3_send_cmd(struct dnbd3_device_t *dev, struct dnbd3_cmd *cmd, int
 	iov.iov_len = sizeof(dnbd3_request);
 	if (kernel_sendmsg(dev->sock, &msg, &iov, 1, sizeof(dnbd3_request)) != sizeof(dnbd3_request)) {
 		printk(KERN_ERR "dnbd3: connection to server lost\n");
-		result = -EAGAIN;
+		result = -EIO;
 		goto error;
 	}
 
@@ -103,11 +103,6 @@ static int dnbd3_send_cmd(struct dnbd3_device_t *dev, struct dnbd3_cmd *cmd, int
 		result = -EIO;
 		goto error;
 	}
-//	if (dnbd3_reply.cmd == 0) {
-//		printk(KERN_ERR "dnbd3: command was 0\n");
-//		result = -EIO;
-//		goto error;
-//	}
 
 	if (dnbd3_reply.cmd != CMD_GET_BLOCK) {
 		printk(KERN_ERR "dnbd3: command was %d\n", dnbd3_reply.cmd);
@@ -146,8 +141,9 @@ static void dnbd3_requeue_cmd(struct dnbd3_cmd *cmd)
 {
 	struct request *req = blk_mq_rq_from_pdu(cmd);
 
-	if (!test_and_set_bit(DNBD3_CMD_REQUEUED, &cmd->flags))
+	if (!test_and_set_bit(DNBD3_CMD_REQUEUED, &cmd->flags)) {
 		blk_mq_requeue_request(req, true);
+	}
 }
 
 static int dnbd3_handle_cmd(struct dnbd3_cmd *cmd, int index)
@@ -222,10 +218,11 @@ static blk_status_t dnbd3_queue_rq(struct blk_mq_hw_ctx *hctx, const struct blk_
 
 
 	ret = dnbd3_handle_cmd(cmd, hctx->queue_num);
-	if (ret < 0)
+	if (ret < 0) {
 		ret = BLK_STS_IOERR;
-	else if (!ret)
+	} else if (!ret) {
 		ret = BLK_STS_OK;
+	}
 	mutex_unlock(&cmd->lock);
 
 	return ret;