added first draft for keepalive and discovery

1 files changed, 290 insertions, 14 deletions

diff --git a/src/kernel/core.c b/src/kernel/core.c
index db1f6de..48f809b 100644
--- a/src/kernel/core.c
+++ b/src/kernel/core.c
@@ -48,27 +48,298 @@
 #include <asm/types.h>
 
 #include "dnbd3.h"
-#include "clientconfig.h"
 #include "sysfs.h"
-#include "block.h"
-#include "mq.h"
+#include "clientconfig.h"
+#include "net.h"
+
+#define DNBD3_CMD_REQUEUED 1
 
 DEFINE_IDR(dnbd3_index_idr);
 DEFINE_MUTEX(dnbd3_index_mutex);
 
 static unsigned int max_devs = NUMBER_DEVICES;
-static dnbd3_device_t *dnbd3_device;
+static dnbd3_device *device;
 int major;
 
 
+static void dnbd3_requeue_cmd(struct dnbd3_cmd *cmd)
+{
+	struct request *req = blk_mq_rq_from_pdu(cmd);
+
+	if (!test_and_set_bit(DNBD3_CMD_REQUEUED, &cmd->flags)) {
+		blk_mq_requeue_request(req, true);
+	}
+}
+
+static int dnbd3_handle_cmd(struct dnbd3_cmd *cmd, int index)
+{
+	struct request *req = blk_mq_rq_from_pdu(cmd);
+	struct dnbd3_device *dev = cmd->dnbd3;
+	struct dnbd3_sock *sock;
+	int ret = -1;
+
+	printk(KERN_DEBUG "dnbd3: handle request at position %lu and size %d, device %i\n", blk_rq_pos(req), blk_rq_bytes(req), dev->minor);
+
+//	if (index >= 1) { // TODO use next server with good rtt for this request
+//		printk(KERN_INFO "dnbd3: index is %d", index);
+//		dev_err_ratelimited(disk_to_dev(dev->disk), "attempted send on invalid socket\n");
+//		blk_mq_start_request(req);
+//		return -EINVAL;
+//	}
+
+	sock = &dev->socks[index];
+	if (!sock->sock) {
+		printk(KERN_INFO "dnbd3: index is %d but no socket was found\n", index);
+		dev_err_ratelimited(disk_to_dev(dev->disk), "attempted send on invalid socket\n");
+		blk_mq_start_request(req);
+		return -EINVAL;
+	}
+
+
+	cmd->status = BLK_STS_OK;
+
+again:
+
+
+	mutex_lock(&sock->lock);
+	if (unlikely(!sock->sock)) {
+		mutex_unlock(&sock->lock);
+		printk(KERN_DEBUG "dnbd3: not connected\n");
+		return -EIO;
+	}
+
+	blk_mq_start_request(req);
+	if (unlikely(sock->pending && sock->pending != req)) {
+		dnbd3_requeue_cmd(cmd);
+		ret = 0;
+		goto out;
+	}
+
+	ret = dnbd3_send_request(dev, sock, blk_mq_rq_from_pdu(cmd));
+	if (ret == -EAGAIN) {
+		dev_err_ratelimited(disk_to_dev(dev->disk), "request send failed, requeueing\n");
+		dnbd3_requeue_cmd(cmd);
+		ret = 0;
+	}
+out:
+	mutex_unlock(&sock->lock);
+	return ret;
+}
+
+static blk_status_t dnbd3_queue_rq(struct blk_mq_hw_ctx *hctx, const struct blk_mq_queue_data *bd)
+{
+	struct dnbd3_cmd *cmd = blk_mq_rq_to_pdu(bd->rq);
+	int ret;
+	struct dnbd3_device *dev = cmd->dnbd3;
+
+	printk(KERN_DEBUG "dnbd3: queue request device %i\n", dev->minor);
+
 
-static int dnbd3_add_device(dnbd3_device_t *dev, int minor)
+	mutex_lock(&cmd->lock);
+	clear_bit(DNBD3_CMD_REQUEUED, &cmd->flags);
+
+
+	ret = dnbd3_handle_cmd(cmd, hctx->queue_num);
+	if (ret < 0) {
+		ret = BLK_STS_IOERR;
+	} else if (!ret) {
+		ret = BLK_STS_OK;
+	}
+	mutex_unlock(&cmd->lock);
+
+	return ret;
+}
+
+static void dnbd3_complete_rq(struct request *req)
+{
+	printk(KERN_DEBUG "dnbd3: dnbd3_complete_rq\n");
+
+}
+
+static int dnbd3_init_request(struct blk_mq_tag_set *set, struct request *rq, unsigned int hctx_idx, unsigned int numa_node)
+{
+	struct dnbd3_cmd *cmd = blk_mq_rq_to_pdu(rq);
+	cmd->dnbd3 = set->driver_data;
+	cmd->flags = 0;
+	mutex_init(&cmd->lock);
+	return 0;
+}
+static enum blk_eh_timer_return dnbd3_xmit_timeout(struct request *req, bool reserved)
+{
+	printk(KERN_DEBUG "dnbd3: dnbd3_xmit_timeout\n");
+	return BLK_EH_DONE;
+}
+
+
+static struct blk_mq_ops dnbd3_mq_ops = {
+	.queue_rq = dnbd3_queue_rq,
+	.complete = dnbd3_complete_rq,
+	.init_request = dnbd3_init_request,
+	.timeout = dnbd3_xmit_timeout,
+};
+
+
+
+
+static void dnbd3_blk_fail_all_requests(dnbd3_device *dev)
+{
+	printk(KERN_DEBUG "dnbd3: fail all requests device %i\n", dev->minor);
+}
+
+
+
+static int dnbd3_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd, unsigned long arg)
+{
+	int result = -100;
+	dnbd3_device *dev = bdev->bd_disk->private_data;
+	char *imgname = NULL;
+	dnbd3_ioctl_t *msg = NULL;
+
+	printk(KERN_DEBUG "dnbd3: ioctl device %i, cmd %i, arg %lu\n", dev->minor, cmd, arg);
+	//unsigned long irqflags;
+
+//	while (dev->disconnecting) {
+//		// do nothing
+//	}
+
+	if (arg != 0) {
+		msg = kmalloc(sizeof(*msg), GFP_KERNEL);
+		if (msg == NULL) return -ENOMEM;
+		if (copy_from_user((char *)msg, (char *)arg, 2) != 0 || msg->len != sizeof(*msg)) {
+			result = -ENOEXEC;
+			goto cleanup_return;
+		}
+		if (copy_from_user((char *)msg, (char *)arg, sizeof(*msg)) != 0) {
+			result = -ENOENT;
+			goto cleanup_return;
+		}
+		if (msg->imgname != NULL && msg->imgnamelen > 0) {
+			imgname = kmalloc(msg->imgnamelen + 1, GFP_KERNEL);
+			if (imgname == NULL) {
+				result = -ENOMEM;
+				goto cleanup_return;
+			}
+			if (copy_from_user(imgname, msg->imgname, msg->imgnamelen) != 0) {
+				result = -ENOENT;
+				goto cleanup_return;
+			}
+			imgname[msg->imgnamelen] = '\0';
+
+			printk(KERN_DEBUG "dnbd3: ioctl image name of len %i is %s\n", (int)msg->imgnamelen, imgname);
+		}
+	}
+
+	mutex_lock(&dev->device_lock);
+	switch (cmd) {
+	case IOCTL_OPEN:
+		printk(KERN_DEBUG "dnbd3: ioctl open\n");
+		if (dev->imgname != NULL) {
+			result = -EBUSY;
+		} else if (imgname == NULL) {
+			result = -EINVAL;
+		} else if (msg == NULL) {
+			result = -EINVAL;
+		} else {
+			if (sizeof(msg->host) != sizeof(dev->initial_server.host)) {
+				printk(KERN_INFO "dnbd3: odd size bug#1 triggered in ioctl\n");
+			}
+			memcpy(&dev->initial_server.host, &msg->host, sizeof(msg->host));
+			dev->initial_server.failures = 0;
+//			memcpy(&dev->initial_server, &dev->cur_server, sizeof(dev->initial_server));
+			dev->imgname = imgname;
+			dev->rid = msg->rid;
+			dev->use_server_provided_alts = msg->use_server_provided_alts;
+			// Forget all alt servers on explicit connect, set first alt server to initial server
+			memset(dev->alt_servers, 0, sizeof(dev->alt_servers[0])*NUMBER_SERVERS);
+			memcpy(dev->alt_servers, &dev->initial_server, sizeof(dev->alt_servers[0]));
+//#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)
+//			if (blk_queue->backing_dev_info != NULL) {
+//				blk_queue->backing_dev_info->ra_pages = (msg->read_ahead_kb * 1024) / PAGE_SIZE;
+//			}
+//#else
+//			blk_queue->backing_dev_info.ra_pages = (msg->read_ahead_kb * 1024) / PAGE_SIZE;
+//#endif
+
+
+			result = dnbd3_net_connect(dev);
+			imgname = NULL;
+		}
+		break;
+
+	case IOCTL_CLOSE:
+		printk(KERN_DEBUG "dnbd3: ioctl close\n");
+		dnbd3_blk_fail_all_requests(dev);
+		dnbd3_net_disconnect(dev);
+		dnbd3_blk_fail_all_requests(dev);
+		set_capacity(dev->disk, 0);
+		if (dev->imgname) {
+			kfree(dev->imgname);
+			dev->imgname = NULL;
+		}
+		break;
+
+	case IOCTL_SWITCH:
+		printk(KERN_DEBUG "dnbd3: ioctl switch\n");
+		result = -EINVAL;
+		break;
+
+	case IOCTL_ADD_SRV:
+	case IOCTL_REM_SRV:
+		printk(KERN_DEBUG "dnbd3: ioctl add/rem srv\n");
+		if (dev->imgname == NULL) {
+			result = -ENOENT;
+		} else if (dev->new_servers_num >= NUMBER_SERVERS) {
+			result = -EAGAIN;
+		} else if (msg == NULL) {
+			result = -EINVAL;
+		} else {
+			memcpy(&dev->new_servers[dev->new_servers_num].host, &msg->host, sizeof(msg->host));
+			dev->new_servers[dev->new_servers_num].failures = (cmd == IOCTL_ADD_SRV ? 0 : 1); // 0 = ADD, 1 = REM
+			++dev->new_servers_num;
+			result = 0;
+		}
+		break;
+
+	case BLKFLSBUF:
+		printk(KERN_DEBUG "dnbd3: ioctl blkflsbuf\n");
+		result = 0;
+		break;
+
+	default:
+		printk(KERN_DEBUG "dnbd3: ioctl unhandled cmd %d\n", cmd);
+		result = -EIO;
+		break;
+	}
+	mutex_unlock(&dev->device_lock);
+cleanup_return:
+	if (msg) kfree(msg);
+	if (imgname) kfree(imgname);
+	return result;
+
+}
+
+
+
+static struct block_device_operations dnbd3_fops =
+{
+	.owner = THIS_MODULE,
+	.ioctl = dnbd3_ioctl,
+	.compat_ioctl =	dnbd3_ioctl,
+};
+
+
+
+
+
+int dnbd3_add_device(dnbd3_device *dev, int minor)
 {
 	struct gendisk *disk;
 	struct request_queue *q;
 	int err = -ENOMEM;
 	printk(KERN_DEBUG "dnbd3: adding device %i\n", minor);
 
+	mutex_init(&dev->device_lock);
+	mutex_lock(&dev->device_lock);
 
 	disk = alloc_disk(1);
 	if (!disk) {
@@ -129,6 +400,9 @@ static int dnbd3_add_device(dnbd3_device_t *dev, int minor)
 	printk(KERN_DEBUG "dnbd3: add disk device %s\n", disk->disk_name);
 	add_disk(disk);
 	dnbd3_sysfs_init(dev);
+
+
+	mutex_unlock(&dev->device_lock);
 	return minor;
 
 out_free_tags:
@@ -139,12 +413,14 @@ out_free_disk:
 	put_disk(disk);
 out_free_dnbd3:
 	kfree(dev);
+	mutex_unlock(&dev->device_lock);
 	printk(KERN_DEBUG "dnbd3: destroy device %i\n", minor);
 	return err;
 }
 
 
 
+
 static int __init dnbd3_init(void)
 {
 	int i;
@@ -156,8 +432,8 @@ static int __init dnbd3_init(void)
 	}
 
 
-	dnbd3_device = kcalloc(max_devs, sizeof(*dnbd3_device), GFP_KERNEL);
-	if (!dnbd3_device) {
+	device = kcalloc(max_devs, sizeof(*device), GFP_KERNEL);
+	if (!device) {
 		printk(KERN_ERR "dnbd3: failed to create dnbd3 device\n");
 		return -ENOMEM;
 	}
@@ -174,7 +450,7 @@ static int __init dnbd3_init(void)
 	// add MAX_NUMBER_DEVICES devices
 	mutex_lock(&dnbd3_index_mutex);
 	for (i = 0; i < max_devs; i++) {
-		dnbd3_add_device(&dnbd3_device[i], i);
+		dnbd3_add_device(&device[i], i);
 	}
 	mutex_unlock(&dnbd3_index_mutex);
 
@@ -187,13 +463,13 @@ static int __init dnbd3_init(void)
 static int dnbd3_exit_cb(int id, void *ptr, void *data)
 {
 	struct list_head *list = (struct list_head *)data;
-	struct dnbd3_device_t *dnbd3 = ptr;
+	struct dnbd3_device *dnbd3 = ptr;
 
 	list_add_tail(&dnbd3->list, list);
 	return 0;
 }
 
-static void dnbd3_dev_remove(struct dnbd3_device_t *dnbd3)
+static void dnbd3_dev_remove(struct dnbd3_device *dnbd3)
 {
 	struct gendisk *disk = dnbd3->disk;
 	struct request_queue *q;
@@ -209,7 +485,7 @@ static void dnbd3_dev_remove(struct dnbd3_device_t *dnbd3)
 	}
 }
 
-static void dnbd3_put(struct dnbd3_device_t *dnbd3)
+static void dnbd3_put(struct dnbd3_device *dnbd3)
 {
 	mutex_lock(&dnbd3_index_mutex);
 	idr_remove(&dnbd3_index_idr, dnbd3->minor);
@@ -220,7 +496,7 @@ static void dnbd3_put(struct dnbd3_device_t *dnbd3)
 
 static void __exit dnbd3_exit(void)
 {
-	dnbd3_device_t *dnbd3;
+	dnbd3_device *dnbd3;
 	LIST_HEAD(del_list);
 	printk(KERN_DEBUG "dnbd3: stopping kernel module\n");
 
@@ -229,7 +505,7 @@ static void __exit dnbd3_exit(void)
 	mutex_unlock(&dnbd3_index_mutex);
 
 	while (!list_empty(&del_list)) {
-		dnbd3 = list_first_entry(&del_list, struct dnbd3_device_t, list);
+		dnbd3 = list_first_entry(&del_list, struct dnbd3_device, list);
 		dnbd3_sysfs_exit(dnbd3);
 		list_del_init(&dnbd3->list);
 		dnbd3_put(dnbd3);
@@ -238,7 +514,7 @@ static void __exit dnbd3_exit(void)
 	idr_destroy(&dnbd3_index_idr);
 	unregister_blkdev(major, "dnbd3");
 
-	kfree(dnbd3_device);
+	kfree(device);
 
 	printk(KERN_INFO "dnbd3: stopped kernel module\n");
 }