added mq

7 files changed, 565 insertions, 244 deletions

diff --git a/Kbuild.in b/Kbuild.in
index ec8b830..1682c83 100644
--- a/Kbuild.in
+++ b/Kbuild.in
@@ -1,2 +1,2 @@
 obj-m := ${MODULE_NAME}.o
-${MODULE_NAME}-objs += core.o sysfs.o utils.o
+${MODULE_NAME}-objs += core.o sysfs.o block.o mq.o utils.o
diff --git a/src/kernel/block.c b/src/kernel/block.c
new file mode 100644
index 0000000..bd2f8d7
--- /dev/null
+++ b/src/kernel/block.c
@@ -0,0 +1,193 @@
+/*
+ * block.c
+ *
+ *  Created on: Jun 26, 2019
+ *      Author: fred
+ */
+
+
+#include "dnbd3.h"
+#include "block.h"
+
+
+
+static int dnbd3_open(struct block_device *bdev, fmode_t mode)
+{
+	dnbd3_device_t *dev = bdev->bd_disk->private_data;
+	printk(KERN_DEBUG "dnbd3: open device %i\n", dev->minor);
+	//TODO can be removed?
+	return 0;
+}
+
+static void dnbd3_release(struct gendisk *disk, fmode_t mode)
+{
+	dnbd3_device_t *dev = disk->private_data;
+	printk(KERN_DEBUG "dnbd3: release device %i\n", dev->minor);
+	//TODO can be removed?
+
+}
+
+
+void dnbd3_blk_fail_all_requests(dnbd3_device_t *dev)
+{
+	printk(KERN_DEBUG "dnbd3: fail all requests device %i\n", dev->minor);
+}
+
+
+int dnbd3_net_connect(dnbd3_device_t *dev)
+{
+	printk(KERN_DEBUG "dnbd3: net connect device %i\n", dev->minor);
+	return 0;
+}
+
+
+int dnbd3_net_disconnect(dnbd3_device_t *dev)
+{
+	printk(KERN_DEBUG "dnbd3: net disconnect device %i\n", dev->minor);
+	return 0;
+}
+
+static int dnbd3_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd, unsigned long arg)
+{
+	int result = -100;
+	dnbd3_device_t *dev = bdev->bd_disk->private_data;
+	char *imgname = NULL;
+	dnbd3_ioctl_t *msg = NULL;
+
+	printk(KERN_DEBUG "dnbd3: ioctl device %i, cmd %i, arg %lu\n", dev->minor, cmd, arg);
+	//unsigned long irqflags;
+
+	while (dev->disconnecting) {
+		// do nothing
+	}
+
+	if (arg != 0) {
+		msg = kmalloc(sizeof(*msg), GFP_KERNEL);
+		if (msg == NULL) return -ENOMEM;
+		if (copy_from_user((char *)msg, (char *)arg, 2) != 0 || msg->len != sizeof(*msg)) {
+			result = -ENOEXEC;
+			goto cleanup_return;
+		}
+		if (copy_from_user((char *)msg, (char *)arg, sizeof(*msg)) != 0) {
+			result = -ENOENT;
+			goto cleanup_return;
+		}
+		if (msg->imgname != NULL && msg->imgnamelen > 0) {
+			imgname = kmalloc(msg->imgnamelen + 1, GFP_KERNEL);
+			if (imgname == NULL) {
+				result = -ENOMEM;
+				goto cleanup_return;
+			}
+			if (copy_from_user(imgname, msg->imgname, msg->imgnamelen) != 0) {
+				result = -ENOENT;
+				goto cleanup_return;
+			}
+			imgname[msg->imgnamelen] = '\0';
+
+			printk(KERN_DEBUG "dnbd3: ioctl image name of len %i is %s\n", (int)msg->imgnamelen, imgname);
+		}
+	}
+
+	switch (cmd) {
+	case IOCTL_OPEN:
+		printk(KERN_DEBUG "dnbd3: ioctl open\n");
+		if (dev->imgname != NULL) {
+			result = -EBUSY;
+		} else if (imgname == NULL) {
+			result = -EINVAL;
+		} else if (msg == NULL) {
+			result = -EINVAL;
+		} else {
+			if (sizeof(msg->host) != sizeof(dev->cur_server.host)) {
+				printk(KERN_INFO "dnbd3: odd size bug#1 triggered in ioctl\n");
+			}
+			memcpy(&dev->cur_server.host, &msg->host, sizeof(msg->host));
+			dev->cur_server.failures = 0;
+			memcpy(&dev->initial_server, &dev->cur_server, sizeof(dev->initial_server));
+			dev->imgname = imgname;
+			dev->rid = msg->rid;
+			dev->use_server_provided_alts = msg->use_server_provided_alts;
+			// Forget all alt servers on explicit connect, set first al server to initial server
+			memset(dev->alt_servers, 0, sizeof(dev->alt_servers[0])*NUMBER_SERVERS);
+			memcpy(dev->alt_servers, &dev->initial_server, sizeof(dev->alt_servers[0]));
+//#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)
+//			if (blk_queue->backing_dev_info != NULL) {
+//				blk_queue->backing_dev_info->ra_pages = (msg->read_ahead_kb * 1024) / PAGE_SIZE;
+//			}
+//#else
+//			blk_queue->backing_dev_info.ra_pages = (msg->read_ahead_kb * 1024) / PAGE_SIZE;
+//#endif
+			if (dnbd3_net_connect(dev) == 0) {
+				result = 0;
+				imgname = NULL; // Prevent kfree at the end
+			} else {
+				result = -ENOENT;
+				dev->imgname = NULL;
+			}
+		}
+		break;
+
+	case IOCTL_CLOSE:
+		printk(KERN_DEBUG "dnbd3: ioctl close\n");
+		dnbd3_blk_fail_all_requests(dev);
+		result = dnbd3_net_disconnect(dev);
+		dnbd3_blk_fail_all_requests(dev);
+		set_capacity(dev->disk, 0);
+		if (dev->imgname) {
+			kfree(dev->imgname);
+			dev->imgname = NULL;
+		}
+		break;
+
+	case IOCTL_SWITCH:
+		printk(KERN_DEBUG "dnbd3: ioctl switch\n");
+		result = -EINVAL;
+		break;
+
+	case IOCTL_ADD_SRV:
+	case IOCTL_REM_SRV:
+		printk(KERN_DEBUG "dnbd3: ioctl add/rem srv\n");
+		if (dev->imgname == NULL) {
+			result = -ENOENT;
+		} else if (dev->new_servers_num >= NUMBER_SERVERS) {
+			result = -EAGAIN;
+		} else if (msg == NULL) {
+			result = -EINVAL;
+		} else {
+			memcpy(&dev->new_servers[dev->new_servers_num].host, &msg->host, sizeof(msg->host));
+			dev->new_servers[dev->new_servers_num].failures = (cmd == IOCTL_ADD_SRV ? 0 : 1); // 0 = ADD, 1 = REM
+			++dev->new_servers_num;
+			result = 0;
+		}
+		break;
+
+	case BLKFLSBUF:
+		printk(KERN_DEBUG "dnbd3: ioctl blkflsbuf\n");
+		result = 0;
+		break;
+
+	default:
+		printk(KERN_DEBUG "dnbd3: ioctl unhandled cmd\n");
+		result = -EIO;
+		break;
+	}
+
+cleanup_return:
+	if (msg) kfree(msg);
+	if (imgname) kfree(imgname);
+	return result;
+
+}
+
+
+
+struct block_device_operations dnbd3_fops =
+{
+	.owner = THIS_MODULE,
+	.open = dnbd3_open,
+	.release = dnbd3_release,
+	.ioctl = dnbd3_ioctl,
+	.compat_ioctl =	dnbd3_ioctl,
+};
+
+
diff --git a/src/kernel/block.h b/src/kernel/block.h
new file mode 100644
index 0000000..5493a00
--- /dev/null
+++ b/src/kernel/block.h
@@ -0,0 +1,14 @@
+/*
+ * block.h
+ *
+ *  Created on: Jun 26, 2019
+ *      Author: fred
+ */
+
+#ifndef SRC_KERNEL_BLOCK_H_
+#define SRC_KERNEL_BLOCK_H_
+
+extern struct block_device_operations dnbd3_fops;
+
+
+#endif /* SRC_KERNEL_BLOCK_H_ */
diff --git a/src/kernel/core.c b/src/kernel/core.c
index bfa8d22..7725a1d 100644
--- a/src/kernel/core.c
+++ b/src/kernel/core.c
@@ -50,6 +50,8 @@
 #include "dnbd3.h"
 #include "clientconfig.h"
 #include "sysfs.h"
+#include "block.h"
+#include "mq.h"
 
 static DEFINE_IDR(dnbd3_index_idr);
 static DEFINE_MUTEX(dnbd3_index_mutex);
@@ -59,249 +61,244 @@ static dnbd3_device_t *dnbd3_device;
 int major;
 
 
-static int dnbd3_open(struct block_device *bdev, fmode_t mode)
-{
-	printk(KERN_DEBUG "dnbd3: dnbd3_open");
-
-	return 0;
-}
-
-static void dnbd3_release(struct gendisk *disk, fmode_t mode)
-{
-	printk(KERN_DEBUG "dnbd3: dnbd3_release");
-
-}
-
-
-void dnbd3_blk_fail_all_requests(dnbd3_device_t *dev)
-{
-	printk(KERN_DEBUG "dnbd3: dnbd3_blk_fail_all_requests");
-}
-
-
-int dnbd3_net_connect(dnbd3_device_t *dev)
-{
-	return 0;
-}
-
-
-int dnbd3_net_disconnect(dnbd3_device_t *dev)
-{
-	return 0;
-}
-
-static int dnbd3_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd, unsigned long arg)
-{
-	printk(KERN_DEBUG "dnbd3: dnbd3_ioctl");
-
-	int result = -100;
-	dnbd3_device_t *dev = bdev->bd_disk->private_data;
-	char *imgname = NULL;
-	dnbd3_ioctl_t *msg = NULL;
-	//unsigned long irqflags;
-
-	while (dev->disconnecting)
-	{
-		// do nothing
-	}
-
-	if (arg != 0)
-	{
-		msg = kmalloc(sizeof(*msg), GFP_KERNEL);
-		if (msg == NULL) return -ENOMEM;
-		if (copy_from_user((char *)msg, (char *)arg, 2) != 0 || msg->len != sizeof(*msg))
-		{
-			result = -ENOEXEC;
-			goto cleanup_return;
-		}
-		if (copy_from_user((char *)msg, (char *)arg, sizeof(*msg)) != 0)
-		{
-			result = -ENOENT;
-			goto cleanup_return;
-		}
-		if (msg->imgname != NULL && msg->imgnamelen > 0)
-		{
-			imgname = kmalloc(msg->imgnamelen + 1, GFP_KERNEL);
-			if (imgname == NULL)
-			{
-				result = -ENOMEM;
-				goto cleanup_return;
-			}
-			if (copy_from_user(imgname, msg->imgname, msg->imgnamelen) != 0)
-			{
-				result = -ENOENT;
-				goto cleanup_return;
-			}
-			imgname[msg->imgnamelen] = '\0';
-			//printk("IOCTL Image name of len %d is %s\n", (int)msg->imgnamelen, imgname);
-		}
-	}
-
-
-	switch (cmd)
-	{
-	case IOCTL_OPEN:
-		if (dev->imgname != NULL)
-		{
-			result = -EBUSY;
-		}
-		else if (imgname == NULL)
-		{
-			result = -EINVAL;
-		}
-		else if (msg == NULL)
-		{
-			result = -EINVAL;
-		}
-		else
-		{
-			if (sizeof(msg->host) != sizeof(dev->cur_server.host))
-				printk("Odd size bug#1 triggered in IOCTL\n");
-			memcpy(&dev->cur_server.host, &msg->host, sizeof(msg->host));
-			dev->cur_server.failures = 0;
-			memcpy(&dev->initial_server, &dev->cur_server, sizeof(dev->initial_server));
-			dev->imgname = imgname;
-			dev->rid = msg->rid;
-			dev->use_server_provided_alts = msg->use_server_provided_alts;
-			// Forget all alt servers on explicit connect, set first al server to initial server
-			memset(dev->alt_servers, 0, sizeof(dev->alt_servers[0])*NUMBER_SERVERS);
-			memcpy(dev->alt_servers, &dev->initial_server, sizeof(dev->alt_servers[0]));
-//#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)
-//			if (blk_queue->backing_dev_info != NULL) {
-//				blk_queue->backing_dev_info->ra_pages = (msg->read_ahead_kb * 1024) / PAGE_SIZE;
+//static int dnbd3_open(struct block_device *bdev, fmode_t mode)
+//{
+//	dnbd3_device_t *dev = bdev->bd_disk->private_data;
+//	printk(KERN_DEBUG "dnbd3: open device %i", dev->minor);
+//
+//	return 0;
+//}
+//
+//static void dnbd3_release(struct gendisk *disk, fmode_t mode)
+//{
+//	dnbd3_device_t *dev = disk->private_data;
+//	printk(KERN_DEBUG "dnbd3: release device %i", dev->minor);
+//
+//}
+//
+//
+//void dnbd3_blk_fail_all_requests(dnbd3_device_t *dev)
+//{
+//	printk(KERN_DEBUG "dnbd3: fail all requests device %i", dev->minor);
+//}
+//
+//
+//int dnbd3_net_connect(dnbd3_device_t *dev)
+//{
+//	printk(KERN_DEBUG "dnbd3: net connect device %i", dev->minor);
+//	return 0;
+//}
+//
+//
+//int dnbd3_net_disconnect(dnbd3_device_t *dev)
+//{
+//	printk(KERN_DEBUG "dnbd3: net disconnect device %i", dev->minor);
+//	return 0;
+//}
+//
+//static int dnbd3_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd, unsigned long arg)
+//{
+//	int result = -100;
+//	dnbd3_device_t *dev = bdev->bd_disk->private_data;
+//	char *imgname = NULL;
+//	dnbd3_ioctl_t *msg = NULL;
+//
+//	printk(KERN_DEBUG "dnbd3: ioctl device %i, cmd %i, arg %lu", dev->minor, cmd, arg);
+//	//unsigned long irqflags;
+//
+//	while (dev->disconnecting) {
+//		// do nothing
+//	}
+//
+//	if (arg != 0) {
+//		msg = kmalloc(sizeof(*msg), GFP_KERNEL);
+//		if (msg == NULL) return -ENOMEM;
+//		if (copy_from_user((char *)msg, (char *)arg, 2) != 0 || msg->len != sizeof(*msg)) {
+//			result = -ENOEXEC;
+//			goto cleanup_return;
+//		}
+//		if (copy_from_user((char *)msg, (char *)arg, sizeof(*msg)) != 0) {
+//			result = -ENOENT;
+//			goto cleanup_return;
+//		}
+//		if (msg->imgname != NULL && msg->imgnamelen > 0) {
+//			imgname = kmalloc(msg->imgnamelen + 1, GFP_KERNEL);
+//			if (imgname == NULL) {
+//				result = -ENOMEM;
+//				goto cleanup_return;
 //			}
-//#else
-//			blk_queue->backing_dev_info.ra_pages = (msg->read_ahead_kb * 1024) / PAGE_SIZE;
-//#endif
-			if (dnbd3_net_connect(dev) == 0)
-			{
-				result = 0;
-				imgname = NULL; // Prevent kfree at the end
-			}
-			else
-			{
-				result = -ENOENT;
-				dev->imgname = NULL;
-			}
-		}
-		break;
-
-	case IOCTL_CLOSE:
-		dnbd3_blk_fail_all_requests(dev);
-		result = dnbd3_net_disconnect(dev);
-		dnbd3_blk_fail_all_requests(dev);
-		set_capacity(dev->disk, 0);
-		if (dev->imgname)
-		{
-			kfree(dev->imgname);
-			dev->imgname = NULL;
-		}
-		break;
-
-	case IOCTL_SWITCH:
-		result = -EINVAL;
-		break;
-
-	case IOCTL_ADD_SRV:
-	case IOCTL_REM_SRV:
-		if (dev->imgname == NULL)
-		{
-			result = -ENOENT;
-		}
-		else if (dev->new_servers_num >= NUMBER_SERVERS)
-		{
-			result = -EAGAIN;
-		}
-		else if (msg == NULL)
-		{
-			result = -EINVAL;
-		}
-		else
-		{
-			memcpy(&dev->new_servers[dev->new_servers_num].host, &msg->host, sizeof(msg->host));
-			dev->new_servers[dev->new_servers_num].failures = (cmd == IOCTL_ADD_SRV ? 0 : 1); // 0 = ADD, 1 = REM
-			++dev->new_servers_num;
-			result = 0;
-		}
-		break;
-
-	case BLKFLSBUF:
-		result = 0;
-		break;
-
-	default:
-		result = -EIO;
-		break;
-	}
-
-cleanup_return:
-	if (msg) kfree(msg);
-	if (imgname) kfree(imgname);
-	return result;
-
-}
-
-static const struct block_device_operations dnbd3_fops =
-{
-	.owner = THIS_MODULE,
-	.open = dnbd3_open,
-	.release = dnbd3_release,
-	.ioctl = dnbd3_ioctl,
-	.compat_ioctl =	dnbd3_ioctl,
-};
-
-static blk_status_t dnbd3_queue_rq(struct blk_mq_hw_ctx *hctx, const struct blk_mq_queue_data *bd)
-{
-	printk(KERN_DEBUG "dnbd3: dnbd3_queue_rq");
-	return 0;
-}
-
-static void dnbd3_complete_rq(struct request *req)
-{
-	printk(KERN_DEBUG "dnbd3: dnbd3_complete_rq");
-
-}
-
-static int dnbd3_init_request(struct blk_mq_tag_set *set, struct request *rq, unsigned int hctx_idx, unsigned int numa_node)
-{
-	struct dnbd3_cmd *cmd = blk_mq_rq_to_pdu(rq);
-	cmd->dnbd3 = set->driver_data;
-	cmd->flags = 0;
-	mutex_init(&cmd->lock);
-	return 0;
-}
-static enum blk_eh_timer_return dnbd3_xmit_timeout(struct request *req, bool reserved)
-{
-	printk(KERN_DEBUG "dnbd3: dnbd3_xmit_timeout");
-	return BLK_EH_DONE;
-}
-
-
-static const struct blk_mq_ops dnbd3_mq_ops = {
-	.queue_rq = dnbd3_queue_rq,
-	.complete = dnbd3_complete_rq,
-	.init_request = dnbd3_init_request,
-	.timeout = dnbd3_xmit_timeout,
-};
-
-
-static int dnbd3_blk_add_device(dnbd3_device_t *dev, int minor)
+//			if (copy_from_user(imgname, msg->imgname, msg->imgnamelen) != 0) {
+//				result = -ENOENT;
+//				goto cleanup_return;
+//			}
+//			imgname[msg->imgnamelen] = '\0';
+//
+//			printk(KERN_DEBUG "dnbd3: ioctl image name of len %i is %s\n", (int)msg->imgnamelen, imgname);
+//		}
+//	}
+//
+//	switch (cmd) {
+//	case IOCTL_OPEN:
+//		printk(KERN_DEBUG "dnbd3: ioctl open");
+//		if (dev->imgname != NULL) {
+//			result = -EBUSY;
+//		} else if (imgname == NULL) {
+//			result = -EINVAL;
+//		} else if (msg == NULL) {
+//			result = -EINVAL;
+//		} else {
+//			if (sizeof(msg->host) != sizeof(dev->cur_server.host)) {
+//				printk(KERN_INFO "dnbd3: odd size bug#1 triggered in ioctl");
+//			}
+//			memcpy(&dev->cur_server.host, &msg->host, sizeof(msg->host));
+//			dev->cur_server.failures = 0;
+//			memcpy(&dev->initial_server, &dev->cur_server, sizeof(dev->initial_server));
+//			dev->imgname = imgname;
+//			dev->rid = msg->rid;
+//			dev->use_server_provided_alts = msg->use_server_provided_alts;
+//			// Forget all alt servers on explicit connect, set first al server to initial server
+//			memset(dev->alt_servers, 0, sizeof(dev->alt_servers[0])*NUMBER_SERVERS);
+//			memcpy(dev->alt_servers, &dev->initial_server, sizeof(dev->alt_servers[0]));
+////#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)
+////			if (blk_queue->backing_dev_info != NULL) {
+////				blk_queue->backing_dev_info->ra_pages = (msg->read_ahead_kb * 1024) / PAGE_SIZE;
+////			}
+////#else
+////			blk_queue->backing_dev_info.ra_pages = (msg->read_ahead_kb * 1024) / PAGE_SIZE;
+////#endif
+//			if (dnbd3_net_connect(dev) == 0) {
+//				result = 0;
+//				imgname = NULL; // Prevent kfree at the end
+//			} else {
+//				result = -ENOENT;
+//				dev->imgname = NULL;
+//			}
+//		}
+//		break;
+//
+//	case IOCTL_CLOSE:
+//		printk(KERN_DEBUG "dnbd3: ioctl close");
+//		dnbd3_blk_fail_all_requests(dev);
+//		result = dnbd3_net_disconnect(dev);
+//		dnbd3_blk_fail_all_requests(dev);
+//		set_capacity(dev->disk, 0);
+//		if (dev->imgname) {
+//			kfree(dev->imgname);
+//			dev->imgname = NULL;
+//		}
+//		break;
+//
+//	case IOCTL_SWITCH:
+//		printk(KERN_DEBUG "dnbd3: ioctl switch");
+//		result = -EINVAL;
+//		break;
+//
+//	case IOCTL_ADD_SRV:
+//	case IOCTL_REM_SRV:
+//		printk(KERN_DEBUG "dnbd3: ioctl add/rem srv");
+//		if (dev->imgname == NULL) {
+//			result = -ENOENT;
+//		} else if (dev->new_servers_num >= NUMBER_SERVERS) {
+//			result = -EAGAIN;
+//		} else if (msg == NULL) {
+//			result = -EINVAL;
+//		} else {
+//			memcpy(&dev->new_servers[dev->new_servers_num].host, &msg->host, sizeof(msg->host));
+//			dev->new_servers[dev->new_servers_num].failures = (cmd == IOCTL_ADD_SRV ? 0 : 1); // 0 = ADD, 1 = REM
+//			++dev->new_servers_num;
+//			result = 0;
+//		}
+//		break;
+//
+//	case BLKFLSBUF:
+//		printk(KERN_DEBUG "dnbd3: ioctl blkflsbuf");
+//		result = 0;
+//		break;
+//
+//	default:
+//		printk(KERN_DEBUG "dnbd3: ioctl unhandled cmd");
+//		result = -EIO;
+//		break;
+//	}
+//
+//cleanup_return:
+//	if (msg) kfree(msg);
+//	if (imgname) kfree(imgname);
+//	return result;
+//
+//}
+//
+//static const struct block_device_operations dnbd3_fops =
+//{
+//	.owner = THIS_MODULE,
+//	.open = dnbd3_open,
+//	.release = dnbd3_release,
+//	.ioctl = dnbd3_ioctl,
+//	.compat_ioctl =	dnbd3_ioctl,
+//};
+
+//static blk_status_t dnbd3_queue_rq(struct blk_mq_hw_ctx *hctx, const struct blk_mq_queue_data *bd)
+//{
+//	struct dnbd3_cmd *cmd = blk_mq_rq_to_pdu(bd->rq);
+//	int ret;
+//	struct dnbd3_device_t *dev;
+//
+//	printk(KERN_DEBUG "dnbd3: queue request device %i\n", dev->minor);
+//
+//	mutex_lock(&cmd->lock);
+//	clear_bit(1, &cmd->flags);
+//
+//
+//
+//	return 0;
+//}
+//
+//static void dnbd3_complete_rq(struct request *req)
+//{
+//	printk(KERN_DEBUG "dnbd3: dnbd3_complete_rq\n");
+//
+//}
+//
+//static int dnbd3_init_request(struct blk_mq_tag_set *set, struct request *rq, unsigned int hctx_idx, unsigned int numa_node)
+//{
+//	struct dnbd3_cmd *cmd = blk_mq_rq_to_pdu(rq);
+//	cmd->dnbd3 = set->driver_data;
+//	cmd->flags = 0;
+//	mutex_init(&cmd->lock);
+//	return 0;
+//}
+//static enum blk_eh_timer_return dnbd3_xmit_timeout(struct request *req, bool reserved)
+//{
+//	printk(KERN_DEBUG "dnbd3: dnbd3_xmit_timeout\n");
+//	return BLK_EH_DONE;
+//}
+//
+//
+//static const struct blk_mq_ops dnbd3_mq_ops = {
+//	.queue_rq = dnbd3_queue_rq,
+//	.complete = dnbd3_complete_rq,
+//	.init_request = dnbd3_init_request,
+//	.timeout = dnbd3_xmit_timeout,
+//};
+
+
+static int dnbd3_add_device(dnbd3_device_t *dev, int minor)
 {
 	struct gendisk *disk;
 	struct request_queue *q;
 	int err = -ENOMEM;
-	printk(KERN_DEBUG "dnbd3: adding device %i", minor);
+	printk(KERN_DEBUG "dnbd3: adding device %i\n", minor);
 
 
 	disk = alloc_disk(1);
 	if (!disk) {
-		printk(KERN_DEBUG "dnbd3: alloc_disc failed, device %i", minor);
+		printk(KERN_WARNING "dnbd3: alloc_disc failed device %i\n", minor);
 		goto out_free_nbd;
 	}
 
 	err = idr_alloc(&dnbd3_index_idr, dev, minor, minor + 1, GFP_KERNEL);
 	if (err == -ENOSPC) {
-		printk(KERN_DEBUG "dnbd3: idr_alloc failed, device %i", minor);
+		printk(KERN_WARNING "dnbd3: idr_alloc failed device %i\n", minor);
 		err = -EEXIST;
 	}
 
@@ -351,9 +348,9 @@ static int dnbd3_blk_add_device(dnbd3_device_t *dev, int minor)
 	disk->first_minor = minor;
 	disk->fops = &dnbd3_fops;
 	disk->private_data = dev;
-	sprintf(disk->disk_name, "dnbd%d", minor);
-//	sprintf(disk->disk_name, "dnbd3%d", minor);
-	printk(KERN_DEBUG "dnbd3: add disk, device %s", disk->disk_name);
+	sprintf(disk->disk_name, "dnbd%i", minor);
+//	sprintf(disk->disk_name, "dnbd3%i", minor);
+	printk(KERN_DEBUG "dnbd3: add disk device %s\n", disk->disk_name);
 	add_disk(disk);
 	dnbd3_sysfs_init(dev);
 	return minor;
@@ -366,7 +363,7 @@ out_free_disk:
 	put_disk(disk);
 out_free_nbd:
 	kfree(dev);
-	printk(KERN_DEBUG "dnbd3: destroy device %i", minor);
+	printk(KERN_DEBUG "dnbd3: destroy device %i\n", minor);
 	return err;
 }
 
@@ -375,33 +372,33 @@ out_free_nbd:
 static int __init dnbd3_init(void)
 {
 	int i;
-	printk(KERN_DEBUG "dnbd3: starting kernel module");
+	printk(KERN_DEBUG "dnbd3: starting kernel module\n");
 
 	if (max_devs < 0) {
-		printk(KERN_ERR "dnbd3: max_devs must be >= 0");
+		printk(KERN_ERR "dnbd3: max_devs must be >= 0\n");
 		return -EINVAL;
 	}
 
 
 	dnbd3_device = kcalloc(max_devs, sizeof(*dnbd3_device), GFP_KERNEL);
 	if (!dnbd3_device) {
-		printk(KERN_ERR "dnbd3: failed to create dnbd3 device");
+		printk(KERN_ERR "dnbd3: failed to create dnbd3 device\n");
 		return -ENOMEM;
 	}
 
 	// initialize block device
 	major = register_blkdev(0, "dnbd3");
 	if (major == 0) {
-		printk(KERN_ERR "dnbd3: register_blkdev failed");
+		printk(KERN_ERR "dnbd3: register_blkdev failed\n");
 		return -EIO;
 	}
 
-	printk(KERN_DEBUG "dnbd3: kernel module loaded. Machine type: " ENDIAN_MODE);
+	printk(KERN_DEBUG "dnbd3: kernel module loaded. Machine type: " ENDIAN_MODE "\n");
 
 	// add MAX_NUMBER_DEVICES devices
 	mutex_lock(&dnbd3_index_mutex);
 	for (i = 0; i < max_devs; i++) {
-		dnbd3_blk_add_device(&dnbd3_device[i], i);
+		dnbd3_add_device(&dnbd3_device[i], i);
 	}
 	mutex_unlock(&dnbd3_index_mutex);
 
@@ -449,7 +446,7 @@ static void __exit dnbd3_exit(void)
 {
 	dnbd3_device_t *dnbd3;
 	LIST_HEAD(del_list);
-	printk(KERN_DEBUG "dnbd3: stopping kernel module");
+	printk(KERN_DEBUG "dnbd3: stopping kernel module\n");
 
 	mutex_lock(&dnbd3_index_mutex);
 	idr_for_each(&dnbd3_index_idr, &dnbd3_exit_cb, &del_list);
@@ -470,7 +467,7 @@ static void __exit dnbd3_exit(void)
 
 	kfree(dnbd3_device);
 
-	printk(KERN_INFO "dnbd3: stopped kernel module");
+	printk(KERN_INFO "dnbd3: stopped kernel module\n");
 }
 
 
diff --git a/src/kernel/dnbd3.h b/src/kernel/dnbd3.h
index 2575cd8..b8b0f59 100644
--- a/src/kernel/dnbd3.h
+++ b/src/kernel/dnbd3.h
@@ -57,7 +57,7 @@ typedef struct dnbd3_device_t {
 	char *imgname;
 	struct socket *sock;
 	dnbd3_server_t cur_server, initial_server;
-	unsigned long cur_rtt;
+	uint64_t cur_rtt;
 //	serialized_buffer_t payload_buffer;
 	dnbd3_server_t alt_servers[NUMBER_SERVERS]; // array of alt servers
 	int new_servers_num;	// number of new alt servers that are waiting to be copied to above array
@@ -70,6 +70,18 @@ typedef struct dnbd3_device_t {
 	// server switch
 	struct socket *better_sock;
 
+
+	// process
+//	struct task_struct * thread_send;
+//	struct task_struct * thread_receive;
+//	struct task_struct *thread_discover;
+//	struct timer_list hb_timer;
+//	wait_queue_head_t process_queue_send;
+//	wait_queue_head_t process_queue_receive;
+//	wait_queue_head_t process_queue_discover;
+//	struct list_head request_queue_send;
+//	struct list_head request_queue_receive;
+
 } dnbd3_device_t;
 
 
@@ -80,7 +92,17 @@ typedef struct dnbd3_cmd {
 	int cookie;
 	blk_status_t status;
 	unsigned long flags;
-	u32 cmd_cookie;
+	uint32_t cmd_cookie;
 } dnbd3_cmd;
 
+typedef struct dnbd3_sock {
+	struct socket *sock;
+	struct mutex tx_lock;
+	struct request *pending;
+	int sent;
+	bool dead;
+	int fallback_index;
+	int cookie;
+} dnbd3_sock;
+
 #endif /* DNBD_H_ */
diff --git a/src/kernel/mq.c b/src/kernel/mq.c
new file mode 100644
index 0000000..d402ecf
--- /dev/null
+++ b/src/kernel/mq.c
@@ -0,0 +1,79 @@
+/*
+ * mq.c
+ *
+ *  Created on: Jun 26, 2019
+ *      Author: fred
+ */
+
+
+#include "mq.h"
+
+#define DNBD3_CMD_REQUEUED 1
+
+
+static int dnbd3_handle_cmd(struct dnbd3_cmd *cmd, int index)
+{
+	struct request *req = blk_mq_rq_from_pdu(cmd);
+	struct dnbd3_device_t *dev = cmd->dnbd3;
+	struct dnbd3_sock *nsock;
+	int ret;
+
+	printk(KERN_DEBUG "dnbd3: handle command %i device %i\n", cmd->cmd_cookie, dev->minor);
+
+
+
+
+	return ret;
+}
+
+static blk_status_t dnbd3_queue_rq(struct blk_mq_hw_ctx *hctx, const struct blk_mq_queue_data *bd)
+{
+	struct dnbd3_cmd *cmd = blk_mq_rq_to_pdu(bd->rq);
+	int ret;
+	struct dnbd3_device_t *dev = cmd->dnbd3;
+
+	printk(KERN_DEBUG "dnbd3: queue request device %i\n", dev->minor);
+
+
+	mutex_lock(&cmd->lock);
+	clear_bit(DNBD3_CMD_REQUEUED, &cmd->flags);
+
+
+	ret = dnbd3_handle_cmd(cmd, hctx->queue_num);
+	if (ret < 0)
+		ret = BLK_STS_IOERR;
+	else if (!ret)
+		ret = BLK_STS_OK;
+	mutex_unlock(&cmd->lock);
+
+	return -10;
+}
+
+static void dnbd3_complete_rq(struct request *req)
+{
+	printk(KERN_DEBUG "dnbd3: dnbd3_complete_rq\n");
+
+}
+
+static int dnbd3_init_request(struct blk_mq_tag_set *set, struct request *rq, unsigned int hctx_idx, unsigned int numa_node)
+{
+	struct dnbd3_cmd *cmd = blk_mq_rq_to_pdu(rq);
+	cmd->dnbd3 = set->driver_data;
+	cmd->flags = 0;
+	mutex_init(&cmd->lock);
+	return 0;
+}
+static enum blk_eh_timer_return dnbd3_xmit_timeout(struct request *req, bool reserved)
+{
+	printk(KERN_DEBUG "dnbd3: dnbd3_xmit_timeout\n");
+	return BLK_EH_DONE;
+}
+
+
+struct blk_mq_ops dnbd3_mq_ops = {
+	.queue_rq = dnbd3_queue_rq,
+	.complete = dnbd3_complete_rq,
+	.init_request = dnbd3_init_request,
+	.timeout = dnbd3_xmit_timeout,
+};
+
diff --git a/src/kernel/mq.h b/src/kernel/mq.h
new file mode 100644
index 0000000..9f3cde8
--- /dev/null
+++ b/src/kernel/mq.h
@@ -0,0 +1,16 @@
+/*
+ * mq.h
+ *
+ *  Created on: Jun 26, 2019
+ *      Author: fred
+ */
+
+#ifndef SRC_KERNEL_MQ_H_
+#define SRC_KERNEL_MQ_H_
+
+#include "dnbd3.h"
+
+extern struct blk_mq_ops dnbd3_mq_ops;
+
+
+#endif /* SRC_KERNEL_MQ_H_ */