From f9ec2db3b4d1e0047087393218618cf8c439c336 Mon Sep 17 00:00:00 2001 From: Frederic Robra Date: Sun, 7 Jul 2019 22:13:01 +0200 Subject: added first draft for keepalive and discovery --- src/kernel/core.c | 304 +++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 290 insertions(+), 14 deletions(-) (limited to 'src/kernel/core.c') diff --git a/src/kernel/core.c b/src/kernel/core.c index db1f6de..48f809b 100644 --- a/src/kernel/core.c +++ b/src/kernel/core.c @@ -48,27 +48,298 @@ #include #include "dnbd3.h" -#include "clientconfig.h" #include "sysfs.h" -#include "block.h" -#include "mq.h" +#include "clientconfig.h" +#include "net.h" + +#define DNBD3_CMD_REQUEUED 1 DEFINE_IDR(dnbd3_index_idr); DEFINE_MUTEX(dnbd3_index_mutex); static unsigned int max_devs = NUMBER_DEVICES; -static dnbd3_device_t *dnbd3_device; +static dnbd3_device *device; int major; +static void dnbd3_requeue_cmd(struct dnbd3_cmd *cmd) +{ + struct request *req = blk_mq_rq_from_pdu(cmd); + + if (!test_and_set_bit(DNBD3_CMD_REQUEUED, &cmd->flags)) { + blk_mq_requeue_request(req, true); + } +} + +static int dnbd3_handle_cmd(struct dnbd3_cmd *cmd, int index) +{ + struct request *req = blk_mq_rq_from_pdu(cmd); + struct dnbd3_device *dev = cmd->dnbd3; + struct dnbd3_sock *sock; + int ret = -1; + + printk(KERN_DEBUG "dnbd3: handle request at position %lu and size %d, device %i\n", blk_rq_pos(req), blk_rq_bytes(req), dev->minor); + +// if (index >= 1) { // TODO use next server with good rtt for this request +// printk(KERN_INFO "dnbd3: index is %d", index); +// dev_err_ratelimited(disk_to_dev(dev->disk), "attempted send on invalid socket\n"); +// blk_mq_start_request(req); +// return -EINVAL; +// } + + sock = &dev->socks[index]; + if (!sock->sock) { + printk(KERN_INFO "dnbd3: index is %d but no socket was found\n", index); + dev_err_ratelimited(disk_to_dev(dev->disk), "attempted send on invalid socket\n"); + blk_mq_start_request(req); + return -EINVAL; + } + + + cmd->status = BLK_STS_OK; + +again: + + + mutex_lock(&sock->lock); + if (unlikely(!sock->sock)) { + mutex_unlock(&sock->lock); + printk(KERN_DEBUG "dnbd3: not connected\n"); + return -EIO; + } + + blk_mq_start_request(req); + if (unlikely(sock->pending && sock->pending != req)) { + dnbd3_requeue_cmd(cmd); + ret = 0; + goto out; + } + + ret = dnbd3_send_request(dev, sock, blk_mq_rq_from_pdu(cmd)); + if (ret == -EAGAIN) { + dev_err_ratelimited(disk_to_dev(dev->disk), "request send failed, requeueing\n"); + dnbd3_requeue_cmd(cmd); + ret = 0; + } +out: + mutex_unlock(&sock->lock); + return ret; +} + +static blk_status_t dnbd3_queue_rq(struct blk_mq_hw_ctx *hctx, const struct blk_mq_queue_data *bd) +{ + struct dnbd3_cmd *cmd = blk_mq_rq_to_pdu(bd->rq); + int ret; + struct dnbd3_device *dev = cmd->dnbd3; + + printk(KERN_DEBUG "dnbd3: queue request device %i\n", dev->minor); + -static int dnbd3_add_device(dnbd3_device_t *dev, int minor) + mutex_lock(&cmd->lock); + clear_bit(DNBD3_CMD_REQUEUED, &cmd->flags); + + + ret = dnbd3_handle_cmd(cmd, hctx->queue_num); + if (ret < 0) { + ret = BLK_STS_IOERR; + } else if (!ret) { + ret = BLK_STS_OK; + } + mutex_unlock(&cmd->lock); + + return ret; +} + +static void dnbd3_complete_rq(struct request *req) +{ + printk(KERN_DEBUG "dnbd3: dnbd3_complete_rq\n"); + +} + +static int dnbd3_init_request(struct blk_mq_tag_set *set, struct request *rq, unsigned int hctx_idx, unsigned int numa_node) +{ + struct dnbd3_cmd *cmd = blk_mq_rq_to_pdu(rq); + cmd->dnbd3 = set->driver_data; + cmd->flags = 0; + mutex_init(&cmd->lock); + return 0; +} +static enum blk_eh_timer_return dnbd3_xmit_timeout(struct request *req, bool reserved) +{ + printk(KERN_DEBUG "dnbd3: dnbd3_xmit_timeout\n"); + return BLK_EH_DONE; +} + + +static struct blk_mq_ops dnbd3_mq_ops = { + .queue_rq = dnbd3_queue_rq, + .complete = dnbd3_complete_rq, + .init_request = dnbd3_init_request, + .timeout = dnbd3_xmit_timeout, +}; + + + + +static void dnbd3_blk_fail_all_requests(dnbd3_device *dev) +{ + printk(KERN_DEBUG "dnbd3: fail all requests device %i\n", dev->minor); +} + + + +static int dnbd3_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd, unsigned long arg) +{ + int result = -100; + dnbd3_device *dev = bdev->bd_disk->private_data; + char *imgname = NULL; + dnbd3_ioctl_t *msg = NULL; + + printk(KERN_DEBUG "dnbd3: ioctl device %i, cmd %i, arg %lu\n", dev->minor, cmd, arg); + //unsigned long irqflags; + +// while (dev->disconnecting) { +// // do nothing +// } + + if (arg != 0) { + msg = kmalloc(sizeof(*msg), GFP_KERNEL); + if (msg == NULL) return -ENOMEM; + if (copy_from_user((char *)msg, (char *)arg, 2) != 0 || msg->len != sizeof(*msg)) { + result = -ENOEXEC; + goto cleanup_return; + } + if (copy_from_user((char *)msg, (char *)arg, sizeof(*msg)) != 0) { + result = -ENOENT; + goto cleanup_return; + } + if (msg->imgname != NULL && msg->imgnamelen > 0) { + imgname = kmalloc(msg->imgnamelen + 1, GFP_KERNEL); + if (imgname == NULL) { + result = -ENOMEM; + goto cleanup_return; + } + if (copy_from_user(imgname, msg->imgname, msg->imgnamelen) != 0) { + result = -ENOENT; + goto cleanup_return; + } + imgname[msg->imgnamelen] = '\0'; + + printk(KERN_DEBUG "dnbd3: ioctl image name of len %i is %s\n", (int)msg->imgnamelen, imgname); + } + } + + mutex_lock(&dev->device_lock); + switch (cmd) { + case IOCTL_OPEN: + printk(KERN_DEBUG "dnbd3: ioctl open\n"); + if (dev->imgname != NULL) { + result = -EBUSY; + } else if (imgname == NULL) { + result = -EINVAL; + } else if (msg == NULL) { + result = -EINVAL; + } else { + if (sizeof(msg->host) != sizeof(dev->initial_server.host)) { + printk(KERN_INFO "dnbd3: odd size bug#1 triggered in ioctl\n"); + } + memcpy(&dev->initial_server.host, &msg->host, sizeof(msg->host)); + dev->initial_server.failures = 0; +// memcpy(&dev->initial_server, &dev->cur_server, sizeof(dev->initial_server)); + dev->imgname = imgname; + dev->rid = msg->rid; + dev->use_server_provided_alts = msg->use_server_provided_alts; + // Forget all alt servers on explicit connect, set first alt server to initial server + memset(dev->alt_servers, 0, sizeof(dev->alt_servers[0])*NUMBER_SERVERS); + memcpy(dev->alt_servers, &dev->initial_server, sizeof(dev->alt_servers[0])); +//#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0) +// if (blk_queue->backing_dev_info != NULL) { +// blk_queue->backing_dev_info->ra_pages = (msg->read_ahead_kb * 1024) / PAGE_SIZE; +// } +//#else +// blk_queue->backing_dev_info.ra_pages = (msg->read_ahead_kb * 1024) / PAGE_SIZE; +//#endif + + + result = dnbd3_net_connect(dev); + imgname = NULL; + } + break; + + case IOCTL_CLOSE: + printk(KERN_DEBUG "dnbd3: ioctl close\n"); + dnbd3_blk_fail_all_requests(dev); + dnbd3_net_disconnect(dev); + dnbd3_blk_fail_all_requests(dev); + set_capacity(dev->disk, 0); + if (dev->imgname) { + kfree(dev->imgname); + dev->imgname = NULL; + } + break; + + case IOCTL_SWITCH: + printk(KERN_DEBUG "dnbd3: ioctl switch\n"); + result = -EINVAL; + break; + + case IOCTL_ADD_SRV: + case IOCTL_REM_SRV: + printk(KERN_DEBUG "dnbd3: ioctl add/rem srv\n"); + if (dev->imgname == NULL) { + result = -ENOENT; + } else if (dev->new_servers_num >= NUMBER_SERVERS) { + result = -EAGAIN; + } else if (msg == NULL) { + result = -EINVAL; + } else { + memcpy(&dev->new_servers[dev->new_servers_num].host, &msg->host, sizeof(msg->host)); + dev->new_servers[dev->new_servers_num].failures = (cmd == IOCTL_ADD_SRV ? 0 : 1); // 0 = ADD, 1 = REM + ++dev->new_servers_num; + result = 0; + } + break; + + case BLKFLSBUF: + printk(KERN_DEBUG "dnbd3: ioctl blkflsbuf\n"); + result = 0; + break; + + default: + printk(KERN_DEBUG "dnbd3: ioctl unhandled cmd %d\n", cmd); + result = -EIO; + break; + } + mutex_unlock(&dev->device_lock); +cleanup_return: + if (msg) kfree(msg); + if (imgname) kfree(imgname); + return result; + +} + + + +static struct block_device_operations dnbd3_fops = +{ + .owner = THIS_MODULE, + .ioctl = dnbd3_ioctl, + .compat_ioctl = dnbd3_ioctl, +}; + + + + + +int dnbd3_add_device(dnbd3_device *dev, int minor) { struct gendisk *disk; struct request_queue *q; int err = -ENOMEM; printk(KERN_DEBUG "dnbd3: adding device %i\n", minor); + mutex_init(&dev->device_lock); + mutex_lock(&dev->device_lock); disk = alloc_disk(1); if (!disk) { @@ -129,6 +400,9 @@ static int dnbd3_add_device(dnbd3_device_t *dev, int minor) printk(KERN_DEBUG "dnbd3: add disk device %s\n", disk->disk_name); add_disk(disk); dnbd3_sysfs_init(dev); + + + mutex_unlock(&dev->device_lock); return minor; out_free_tags: @@ -139,12 +413,14 @@ out_free_disk: put_disk(disk); out_free_dnbd3: kfree(dev); + mutex_unlock(&dev->device_lock); printk(KERN_DEBUG "dnbd3: destroy device %i\n", minor); return err; } + static int __init dnbd3_init(void) { int i; @@ -156,8 +432,8 @@ static int __init dnbd3_init(void) } - dnbd3_device = kcalloc(max_devs, sizeof(*dnbd3_device), GFP_KERNEL); - if (!dnbd3_device) { + device = kcalloc(max_devs, sizeof(*device), GFP_KERNEL); + if (!device) { printk(KERN_ERR "dnbd3: failed to create dnbd3 device\n"); return -ENOMEM; } @@ -174,7 +450,7 @@ static int __init dnbd3_init(void) // add MAX_NUMBER_DEVICES devices mutex_lock(&dnbd3_index_mutex); for (i = 0; i < max_devs; i++) { - dnbd3_add_device(&dnbd3_device[i], i); + dnbd3_add_device(&device[i], i); } mutex_unlock(&dnbd3_index_mutex); @@ -187,13 +463,13 @@ static int __init dnbd3_init(void) static int dnbd3_exit_cb(int id, void *ptr, void *data) { struct list_head *list = (struct list_head *)data; - struct dnbd3_device_t *dnbd3 = ptr; + struct dnbd3_device *dnbd3 = ptr; list_add_tail(&dnbd3->list, list); return 0; } -static void dnbd3_dev_remove(struct dnbd3_device_t *dnbd3) +static void dnbd3_dev_remove(struct dnbd3_device *dnbd3) { struct gendisk *disk = dnbd3->disk; struct request_queue *q; @@ -209,7 +485,7 @@ static void dnbd3_dev_remove(struct dnbd3_device_t *dnbd3) } } -static void dnbd3_put(struct dnbd3_device_t *dnbd3) +static void dnbd3_put(struct dnbd3_device *dnbd3) { mutex_lock(&dnbd3_index_mutex); idr_remove(&dnbd3_index_idr, dnbd3->minor); @@ -220,7 +496,7 @@ static void dnbd3_put(struct dnbd3_device_t *dnbd3) static void __exit dnbd3_exit(void) { - dnbd3_device_t *dnbd3; + dnbd3_device *dnbd3; LIST_HEAD(del_list); printk(KERN_DEBUG "dnbd3: stopping kernel module\n"); @@ -229,7 +505,7 @@ static void __exit dnbd3_exit(void) mutex_unlock(&dnbd3_index_mutex); while (!list_empty(&del_list)) { - dnbd3 = list_first_entry(&del_list, struct dnbd3_device_t, list); + dnbd3 = list_first_entry(&del_list, struct dnbd3_device, list); dnbd3_sysfs_exit(dnbd3); list_del_init(&dnbd3->list); dnbd3_put(dnbd3); @@ -238,7 +514,7 @@ static void __exit dnbd3_exit(void) idr_destroy(&dnbd3_index_idr); unregister_blkdev(major, "dnbd3"); - kfree(dnbd3_device); + kfree(device); printk(KERN_INFO "dnbd3: stopped kernel module\n"); } -- cgit v1.2.3-55-g7522