From e21031a425e568886b0be8048318ac7ded1616b7 Mon Sep 17 00:00:00 2001 From: Frederic Robra Date: Fri, 5 Jul 2019 15:32:13 +0200 Subject: fixed some errors with multi queue --- src/kernel/block.c | 45 ++++++----- src/kernel/core.c | 226 +---------------------------------------------------- src/kernel/dnbd3.h | 21 +++-- src/kernel/mq.c | 37 ++++----- 4 files changed, 54 insertions(+), 275 deletions(-) (limited to 'src') diff --git a/src/kernel/block.c b/src/kernel/block.c index 26caa25..ade02f0 100644 --- a/src/kernel/block.c +++ b/src/kernel/block.c @@ -15,22 +15,6 @@ -static int dnbd3_open(struct block_device *bdev, fmode_t mode) -{ - dnbd3_device_t *dev = bdev->bd_disk->private_data; - printk(KERN_DEBUG "dnbd3: open device %i\n", dev->minor); - //TODO can be removed? - return 0; -} - -static void dnbd3_release(struct gendisk *disk, fmode_t mode) -{ - dnbd3_device_t *dev = disk->private_data; - printk(KERN_DEBUG "dnbd3: release device %i\n", dev->minor); - //TODO can be removed? - -} - void dnbd3_blk_fail_all_requests(dnbd3_device_t *dev) { @@ -51,7 +35,7 @@ int dnbd3_net_connect(dnbd3_device_t *dev) while (dev->disconnecting) schedule(); } - if (dev->request) { + if (dev->pending) { printk(KERN_DEBUG "dnbd3: device still in request\n"); while (dev->disconnecting) schedule(); @@ -217,7 +201,7 @@ int dnbd3_net_connect(dnbd3_device_t *dev) dev->panic_count = 0; refcount_inc(&dev->config_refs); - + mutex_init(&dev->socket_lock); // Enqueue request to request_queue_send for a fresh list of alt servers //TODO refresh alt server list @@ -273,6 +257,26 @@ int dnbd3_net_disconnect(dnbd3_device_t *dev) return 0; } +static void printHost(struct dnbd3_host_t *host, char *msg) +{ + if (host->type == HOST_IP4) { + printk(KERN_INFO "dnbd3: %s %pI4:%d", msg, host->addr, host->port); + } else { + printk(KERN_INFO "dnbd3: %s [%pI6]:%d", msg, host->addr, host->port); + } +} + +static void printServerList(struct dnbd3_device_t *dev) +{ + int i; + printHost(&dev->cur_server.host, "current server is"); + for (i = 0; i < NUMBER_SERVERS; i++) { + if (dev->alt_servers[i].host.addr[0] != 0) { + printHost(&dev->alt_servers[i].host, "alternative server is"); + } + } +} + static int dnbd3_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd, unsigned long arg) { int result = -100; @@ -343,6 +347,9 @@ static int dnbd3_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd //#else // blk_queue->backing_dev_info.ra_pages = (msg->read_ahead_kb * 1024) / PAGE_SIZE; //#endif + + printServerList(dev); + if (dnbd3_net_connect(dev) == 0) { result = 0; imgname = NULL; // Prevent kfree at the end @@ -410,8 +417,6 @@ cleanup_return: struct block_device_operations dnbd3_fops = { .owner = THIS_MODULE, - .open = dnbd3_open, - .release = dnbd3_release, .ioctl = dnbd3_ioctl, .compat_ioctl = dnbd3_ioctl, }; diff --git a/src/kernel/core.c b/src/kernel/core.c index 731c094..e92bd9a 100644 --- a/src/kernel/core.c +++ b/src/kernel/core.c @@ -61,226 +61,6 @@ static dnbd3_device_t *dnbd3_device; int major; -//static int dnbd3_open(struct block_device *bdev, fmode_t mode) -//{ -// dnbd3_device_t *dev = bdev->bd_disk->private_data; -// printk(KERN_DEBUG "dnbd3: open device %i", dev->minor); -// -// return 0; -//} -// -//static void dnbd3_release(struct gendisk *disk, fmode_t mode) -//{ -// dnbd3_device_t *dev = disk->private_data; -// printk(KERN_DEBUG "dnbd3: release device %i", dev->minor); -// -//} -// -// -//void dnbd3_blk_fail_all_requests(dnbd3_device_t *dev) -//{ -// printk(KERN_DEBUG "dnbd3: fail all requests device %i", dev->minor); -//} -// -// -//int dnbd3_net_connect(dnbd3_device_t *dev) -//{ -// printk(KERN_DEBUG "dnbd3: net connect device %i", dev->minor); -// return 0; -//} -// -// -//int dnbd3_net_disconnect(dnbd3_device_t *dev) -//{ -// printk(KERN_DEBUG "dnbd3: net disconnect device %i", dev->minor); -// return 0; -//} -// -//static int dnbd3_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd, unsigned long arg) -//{ -// int result = -100; -// dnbd3_device_t *dev = bdev->bd_disk->private_data; -// char *imgname = NULL; -// dnbd3_ioctl_t *msg = NULL; -// -// printk(KERN_DEBUG "dnbd3: ioctl device %i, cmd %i, arg %lu", dev->minor, cmd, arg); -// //unsigned long irqflags; -// -// while (dev->disconnecting) { -// // do nothing -// } -// -// if (arg != 0) { -// msg = kmalloc(sizeof(*msg), GFP_KERNEL); -// if (msg == NULL) return -ENOMEM; -// if (copy_from_user((char *)msg, (char *)arg, 2) != 0 || msg->len != sizeof(*msg)) { -// result = -ENOEXEC; -// goto cleanup_return; -// } -// if (copy_from_user((char *)msg, (char *)arg, sizeof(*msg)) != 0) { -// result = -ENOENT; -// goto cleanup_return; -// } -// if (msg->imgname != NULL && msg->imgnamelen > 0) { -// imgname = kmalloc(msg->imgnamelen + 1, GFP_KERNEL); -// if (imgname == NULL) { -// result = -ENOMEM; -// goto cleanup_return; -// } -// if (copy_from_user(imgname, msg->imgname, msg->imgnamelen) != 0) { -// result = -ENOENT; -// goto cleanup_return; -// } -// imgname[msg->imgnamelen] = '\0'; -// -// printk(KERN_DEBUG "dnbd3: ioctl image name of len %i is %s\n", (int)msg->imgnamelen, imgname); -// } -// } -// -// switch (cmd) { -// case IOCTL_OPEN: -// printk(KERN_DEBUG "dnbd3: ioctl open"); -// if (dev->imgname != NULL) { -// result = -EBUSY; -// } else if (imgname == NULL) { -// result = -EINVAL; -// } else if (msg == NULL) { -// result = -EINVAL; -// } else { -// if (sizeof(msg->host) != sizeof(dev->cur_server.host)) { -// printk(KERN_INFO "dnbd3: odd size bug#1 triggered in ioctl"); -// } -// memcpy(&dev->cur_server.host, &msg->host, sizeof(msg->host)); -// dev->cur_server.failures = 0; -// memcpy(&dev->initial_server, &dev->cur_server, sizeof(dev->initial_server)); -// dev->imgname = imgname; -// dev->rid = msg->rid; -// dev->use_server_provided_alts = msg->use_server_provided_alts; -// // Forget all alt servers on explicit connect, set first al server to initial server -// memset(dev->alt_servers, 0, sizeof(dev->alt_servers[0])*NUMBER_SERVERS); -// memcpy(dev->alt_servers, &dev->initial_server, sizeof(dev->alt_servers[0])); -////#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0) -//// if (blk_queue->backing_dev_info != NULL) { -//// blk_queue->backing_dev_info->ra_pages = (msg->read_ahead_kb * 1024) / PAGE_SIZE; -//// } -////#else -//// blk_queue->backing_dev_info.ra_pages = (msg->read_ahead_kb * 1024) / PAGE_SIZE; -////#endif -// if (dnbd3_net_connect(dev) == 0) { -// result = 0; -// imgname = NULL; // Prevent kfree at the end -// } else { -// result = -ENOENT; -// dev->imgname = NULL; -// } -// } -// break; -// -// case IOCTL_CLOSE: -// printk(KERN_DEBUG "dnbd3: ioctl close"); -// dnbd3_blk_fail_all_requests(dev); -// result = dnbd3_net_disconnect(dev); -// dnbd3_blk_fail_all_requests(dev); -// set_capacity(dev->disk, 0); -// if (dev->imgname) { -// kfree(dev->imgname); -// dev->imgname = NULL; -// } -// break; -// -// case IOCTL_SWITCH: -// printk(KERN_DEBUG "dnbd3: ioctl switch"); -// result = -EINVAL; -// break; -// -// case IOCTL_ADD_SRV: -// case IOCTL_REM_SRV: -// printk(KERN_DEBUG "dnbd3: ioctl add/rem srv"); -// if (dev->imgname == NULL) { -// result = -ENOENT; -// } else if (dev->new_servers_num >= NUMBER_SERVERS) { -// result = -EAGAIN; -// } else if (msg == NULL) { -// result = -EINVAL; -// } else { -// memcpy(&dev->new_servers[dev->new_servers_num].host, &msg->host, sizeof(msg->host)); -// dev->new_servers[dev->new_servers_num].failures = (cmd == IOCTL_ADD_SRV ? 0 : 1); // 0 = ADD, 1 = REM -// ++dev->new_servers_num; -// result = 0; -// } -// break; -// -// case BLKFLSBUF: -// printk(KERN_DEBUG "dnbd3: ioctl blkflsbuf"); -// result = 0; -// break; -// -// default: -// printk(KERN_DEBUG "dnbd3: ioctl unhandled cmd"); -// result = -EIO; -// break; -// } -// -//cleanup_return: -// if (msg) kfree(msg); -// if (imgname) kfree(imgname); -// return result; -// -//} -// -//static const struct block_device_operations dnbd3_fops = -//{ -// .owner = THIS_MODULE, -// .open = dnbd3_open, -// .release = dnbd3_release, -// .ioctl = dnbd3_ioctl, -// .compat_ioctl = dnbd3_ioctl, -//}; - -//static blk_status_t dnbd3_queue_rq(struct blk_mq_hw_ctx *hctx, const struct blk_mq_queue_data *bd) -//{ -// struct dnbd3_cmd *cmd = blk_mq_rq_to_pdu(bd->rq); -// int ret; -// struct dnbd3_device_t *dev; -// -// printk(KERN_DEBUG "dnbd3: queue request device %i\n", dev->minor); -// -// mutex_lock(&cmd->lock); -// clear_bit(1, &cmd->flags); -// -// -// -// return 0; -//} -// -//static void dnbd3_complete_rq(struct request *req) -//{ -// printk(KERN_DEBUG "dnbd3: dnbd3_complete_rq\n"); -// -//} -// -//static int dnbd3_init_request(struct blk_mq_tag_set *set, struct request *rq, unsigned int hctx_idx, unsigned int numa_node) -//{ -// struct dnbd3_cmd *cmd = blk_mq_rq_to_pdu(rq); -// cmd->dnbd3 = set->driver_data; -// cmd->flags = 0; -// mutex_init(&cmd->lock); -// return 0; -//} -//static enum blk_eh_timer_return dnbd3_xmit_timeout(struct request *req, bool reserved) -//{ -// printk(KERN_DEBUG "dnbd3: dnbd3_xmit_timeout\n"); -// return BLK_EH_DONE; -//} -// -// -//static const struct blk_mq_ops dnbd3_mq_ops = { -// .queue_rq = dnbd3_queue_rq, -// .complete = dnbd3_complete_rq, -// .init_request = dnbd3_init_request, -// .timeout = dnbd3_xmit_timeout, -//}; - static int dnbd3_add_device(dnbd3_device_t *dev, int minor) { @@ -293,7 +73,7 @@ static int dnbd3_add_device(dnbd3_device_t *dev, int minor) disk = alloc_disk(1); if (!disk) { printk(KERN_WARNING "dnbd3: alloc_disc failed device %i\n", minor); - goto out_free_nbd; + goto out_free_dnbd3; } err = idr_alloc(&dnbd3_index_idr, dev, minor, minor + 1, GFP_KERNEL); @@ -308,7 +88,7 @@ static int dnbd3_add_device(dnbd3_device_t *dev, int minor) dev->minor = minor; dev->disk = disk; dev->tag_set.ops = &dnbd3_mq_ops; - dev->tag_set.nr_hw_queues = 1; + dev->tag_set.nr_hw_queues = 1; // this can be changed later with blk_mq_update_nr_hw_queues() dev->tag_set.queue_depth = 128; dev->tag_set.numa_node = NUMA_NO_NODE; dev->tag_set.cmd_size = sizeof(dnbd3_cmd); @@ -361,7 +141,7 @@ out_free_idr: idr_remove(&dnbd3_index_idr, minor); out_free_disk: put_disk(disk); -out_free_nbd: +out_free_dnbd3: kfree(dev); printk(KERN_DEBUG "dnbd3: destroy device %i\n", minor); return err; diff --git a/src/kernel/dnbd3.h b/src/kernel/dnbd3.h index af9bdff..0a4ef67 100644 --- a/src/kernel/dnbd3.h +++ b/src/kernel/dnbd3.h @@ -44,7 +44,7 @@ typedef struct typedef struct dnbd3_device_t { int minor; struct blk_mq_tag_set tag_set; - struct request_queue queue; +// struct request_queue queue; struct mutex config_lock; refcount_t config_refs; refcount_t refs; @@ -67,7 +67,7 @@ typedef struct dnbd3_device_t { dnbd3_server_t alt_servers[NUMBER_SERVERS]; // array of alt servers int new_servers_num; // number of new alt servers that are waiting to be copied to above array dnbd3_server_entry_t new_servers[NUMBER_SERVERS]; // pending new alt servers - uint8_t discover, panic, disconnecting, update_available, panic_count, request; + uint8_t discover, panic, disconnecting, update_available, panic_count; uint8_t use_server_provided_alts; uint16_t rid; uint32_t heartbeat_count; @@ -99,15 +99,12 @@ typedef struct dnbd3_cmd { unsigned long flags; uint32_t cmd_cookie; } dnbd3_cmd; -// -//typedef struct dnbd3_sock { -// struct socket *sock; -// struct mutex tx_lock; -// struct request *pending; -// int sent; -// bool dead; -// int fallback_index; -// int cookie; -//} dnbd3_sock; + +typedef struct dnbd3_sock { + struct socket *sock; + struct mutex lock; + struct request *pending; + struct dnbd3_server_t *server; +} dnbd3_sock; #endif /* DNBD_H_ */ diff --git a/src/kernel/mq.c b/src/kernel/mq.c index e3407ab..3b0bde5 100644 --- a/src/kernel/mq.c +++ b/src/kernel/mq.c @@ -11,7 +11,7 @@ #include "block.h" #define DNBD3_CMD_REQUEUED 1 - +// //static void dnbd3_config_put(struct dnbd3_device_t *dev) //{ // if (refcount_dec_and_mutex_lock(&dev->config_refs, @@ -44,21 +44,21 @@ static int dnbd3_send_cmd(struct dnbd3_device_t *dev, struct dnbd3_cmd *cmd, int sigset_t blocked, oldset; void *kaddr; int result; - + dev->pending = req; init_msghdr(msg); dnbd3_request.magic = dnbd3_packet_magic; switch (req_op(req)) { - case REQ_OP_DISCARD: - printk(KERN_DEBUG "dnbd3: request operation discard on device %d\n", dev->minor); - break; - case REQ_OP_FLUSH: - printk(KERN_DEBUG "dnbd3: request operation flush on device %d\n", dev->minor); - break; - case REQ_OP_WRITE: - printk(KERN_DEBUG "dnbd3: request operation write on device %d\n", dev->minor); - break; +// case REQ_OP_DISCARD: +// printk(KERN_DEBUG "dnbd3: request operation discard on device %d\n", dev->minor); +// break; +// case REQ_OP_FLUSH: +// printk(KERN_DEBUG "dnbd3: request operation flush on device %d\n", dev->minor); +// break; +// case REQ_OP_WRITE: +// printk(KERN_DEBUG "dnbd3: request operation write on device %d\n", dev->minor); +// break; case REQ_OP_READ: printk(KERN_DEBUG "dnbd3: request operation read on device %d\n", dev->minor); dnbd3_request.cmd = CMD_GET_BLOCK; @@ -81,7 +81,7 @@ static int dnbd3_send_cmd(struct dnbd3_device_t *dev, struct dnbd3_cmd *cmd, int iov.iov_len = sizeof(dnbd3_request); if (kernel_sendmsg(dev->sock, &msg, &iov, 1, sizeof(dnbd3_request)) != sizeof(dnbd3_request)) { printk(KERN_ERR "dnbd3: connection to server lost\n"); - result = -EAGAIN; + result = -EIO; goto error; } @@ -103,11 +103,6 @@ static int dnbd3_send_cmd(struct dnbd3_device_t *dev, struct dnbd3_cmd *cmd, int result = -EIO; goto error; } -// if (dnbd3_reply.cmd == 0) { -// printk(KERN_ERR "dnbd3: command was 0\n"); -// result = -EIO; -// goto error; -// } if (dnbd3_reply.cmd != CMD_GET_BLOCK) { printk(KERN_ERR "dnbd3: command was %d\n", dnbd3_reply.cmd); @@ -146,8 +141,9 @@ static void dnbd3_requeue_cmd(struct dnbd3_cmd *cmd) { struct request *req = blk_mq_rq_from_pdu(cmd); - if (!test_and_set_bit(DNBD3_CMD_REQUEUED, &cmd->flags)) + if (!test_and_set_bit(DNBD3_CMD_REQUEUED, &cmd->flags)) { blk_mq_requeue_request(req, true); + } } static int dnbd3_handle_cmd(struct dnbd3_cmd *cmd, int index) @@ -222,10 +218,11 @@ static blk_status_t dnbd3_queue_rq(struct blk_mq_hw_ctx *hctx, const struct blk_ ret = dnbd3_handle_cmd(cmd, hctx->queue_num); - if (ret < 0) + if (ret < 0) { ret = BLK_STS_IOERR; - else if (!ret) + } else if (!ret) { ret = BLK_STS_OK; + } mutex_unlock(&cmd->lock); return ret; -- cgit v1.2.3-55-g7522