first version of dnbd3-ng

1 files changed, 484 insertions, 0 deletions

diff --git a/src/kernel/core.c b/src/kernel/core.c
new file mode 100644
index 0000000..bfa8d22
--- /dev/null
+++ b/src/kernel/core.c
@@ -0,0 +1,484 @@
+/*
+ * This file is part of the Distributed Network Block Device 3
+ *
+ * Copyright(c) 2019 Frederic Robra <frederic@robra.org>
+ * Parts copyright 2011-2012 Johann Latocha <johann@latocha.de>
+ *
+ * This file may be licensed under the terms of of the
+ * GNU General Public License Version 2 (the ``GPL'').
+ *
+ * Software distributed under the License is distributed
+ * on an ``AS IS'' basis, WITHOUT WARRANTY OF ANY KIND, either
+ * express or implied. See the GPL for the specific language
+ * governing rights and limitations.
+ *
+ * You should have received a copy of the GPL along with this
+ * program. If not, go to http://www.gnu.org/licenses/gpl.html
+ * or write to the Free Software Foundation, Inc.,
+ * 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ *
+ */
+
+#include <linux/major.h>
+
+#include <linux/blkdev.h>
+#include <linux/module.h>
+#include <linux/init.h>
+#include <linux/sched.h>
+#include <linux/sched/mm.h>
+#include <linux/fs.h>
+#include <linux/bio.h>
+#include <linux/stat.h>
+#include <linux/errno.h>
+#include <linux/file.h>
+#include <linux/ioctl.h>
+#include <linux/mutex.h>
+#include <linux/compiler.h>
+#include <linux/err.h>
+#include <linux/kernel.h>
+#include <linux/slab.h>
+#include <net/sock.h>
+#include <linux/net.h>
+#include <linux/kthread.h>
+#include <linux/types.h>
+#include <linux/debugfs.h>
+#include <linux/blk-mq.h>
+
+#include <linux/uaccess.h>
+#include <asm/types.h>
+
+#include "dnbd3.h"
+#include "clientconfig.h"
+#include "sysfs.h"
+
+static DEFINE_IDR(dnbd3_index_idr);
+static DEFINE_MUTEX(dnbd3_index_mutex);
+
+static unsigned int max_devs = NUMBER_DEVICES;
+static dnbd3_device_t *dnbd3_device;
+int major;
+
+
+static int dnbd3_open(struct block_device *bdev, fmode_t mode)
+{
+	printk(KERN_DEBUG "dnbd3: dnbd3_open");
+
+	return 0;
+}
+
+static void dnbd3_release(struct gendisk *disk, fmode_t mode)
+{
+	printk(KERN_DEBUG "dnbd3: dnbd3_release");
+
+}
+
+
+void dnbd3_blk_fail_all_requests(dnbd3_device_t *dev)
+{
+	printk(KERN_DEBUG "dnbd3: dnbd3_blk_fail_all_requests");
+}
+
+
+int dnbd3_net_connect(dnbd3_device_t *dev)
+{
+	return 0;
+}
+
+
+int dnbd3_net_disconnect(dnbd3_device_t *dev)
+{
+	return 0;
+}
+
+static int dnbd3_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd, unsigned long arg)
+{
+	printk(KERN_DEBUG "dnbd3: dnbd3_ioctl");
+
+	int result = -100;
+	dnbd3_device_t *dev = bdev->bd_disk->private_data;
+	char *imgname = NULL;
+	dnbd3_ioctl_t *msg = NULL;
+	//unsigned long irqflags;
+
+	while (dev->disconnecting)
+	{
+		// do nothing
+	}
+
+	if (arg != 0)
+	{
+		msg = kmalloc(sizeof(*msg), GFP_KERNEL);
+		if (msg == NULL) return -ENOMEM;
+		if (copy_from_user((char *)msg, (char *)arg, 2) != 0 || msg->len != sizeof(*msg))
+		{
+			result = -ENOEXEC;
+			goto cleanup_return;
+		}
+		if (copy_from_user((char *)msg, (char *)arg, sizeof(*msg)) != 0)
+		{
+			result = -ENOENT;
+			goto cleanup_return;
+		}
+		if (msg->imgname != NULL && msg->imgnamelen > 0)
+		{
+			imgname = kmalloc(msg->imgnamelen + 1, GFP_KERNEL);
+			if (imgname == NULL)
+			{
+				result = -ENOMEM;
+				goto cleanup_return;
+			}
+			if (copy_from_user(imgname, msg->imgname, msg->imgnamelen) != 0)
+			{
+				result = -ENOENT;
+				goto cleanup_return;
+			}
+			imgname[msg->imgnamelen] = '\0';
+			//printk("IOCTL Image name of len %d is %s\n", (int)msg->imgnamelen, imgname);
+		}
+	}
+
+
+	switch (cmd)
+	{
+	case IOCTL_OPEN:
+		if (dev->imgname != NULL)
+		{
+			result = -EBUSY;
+		}
+		else if (imgname == NULL)
+		{
+			result = -EINVAL;
+		}
+		else if (msg == NULL)
+		{
+			result = -EINVAL;
+		}
+		else
+		{
+			if (sizeof(msg->host) != sizeof(dev->cur_server.host))
+				printk("Odd size bug#1 triggered in IOCTL\n");
+			memcpy(&dev->cur_server.host, &msg->host, sizeof(msg->host));
+			dev->cur_server.failures = 0;
+			memcpy(&dev->initial_server, &dev->cur_server, sizeof(dev->initial_server));
+			dev->imgname = imgname;
+			dev->rid = msg->rid;
+			dev->use_server_provided_alts = msg->use_server_provided_alts;
+			// Forget all alt servers on explicit connect, set first al server to initial server
+			memset(dev->alt_servers, 0, sizeof(dev->alt_servers[0])*NUMBER_SERVERS);
+			memcpy(dev->alt_servers, &dev->initial_server, sizeof(dev->alt_servers[0]));
+//#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 11, 0)
+//			if (blk_queue->backing_dev_info != NULL) {
+//				blk_queue->backing_dev_info->ra_pages = (msg->read_ahead_kb * 1024) / PAGE_SIZE;
+//			}
+//#else
+//			blk_queue->backing_dev_info.ra_pages = (msg->read_ahead_kb * 1024) / PAGE_SIZE;
+//#endif
+			if (dnbd3_net_connect(dev) == 0)
+			{
+				result = 0;
+				imgname = NULL; // Prevent kfree at the end
+			}
+			else
+			{
+				result = -ENOENT;
+				dev->imgname = NULL;
+			}
+		}
+		break;
+
+	case IOCTL_CLOSE:
+		dnbd3_blk_fail_all_requests(dev);
+		result = dnbd3_net_disconnect(dev);
+		dnbd3_blk_fail_all_requests(dev);
+		set_capacity(dev->disk, 0);
+		if (dev->imgname)
+		{
+			kfree(dev->imgname);
+			dev->imgname = NULL;
+		}
+		break;
+
+	case IOCTL_SWITCH:
+		result = -EINVAL;
+		break;
+
+	case IOCTL_ADD_SRV:
+	case IOCTL_REM_SRV:
+		if (dev->imgname == NULL)
+		{
+			result = -ENOENT;
+		}
+		else if (dev->new_servers_num >= NUMBER_SERVERS)
+		{
+			result = -EAGAIN;
+		}
+		else if (msg == NULL)
+		{
+			result = -EINVAL;
+		}
+		else
+		{
+			memcpy(&dev->new_servers[dev->new_servers_num].host, &msg->host, sizeof(msg->host));
+			dev->new_servers[dev->new_servers_num].failures = (cmd == IOCTL_ADD_SRV ? 0 : 1); // 0 = ADD, 1 = REM
+			++dev->new_servers_num;
+			result = 0;
+		}
+		break;
+
+	case BLKFLSBUF:
+		result = 0;
+		break;
+
+	default:
+		result = -EIO;
+		break;
+	}
+
+cleanup_return:
+	if (msg) kfree(msg);
+	if (imgname) kfree(imgname);
+	return result;
+
+}
+
+static const struct block_device_operations dnbd3_fops =
+{
+	.owner = THIS_MODULE,
+	.open = dnbd3_open,
+	.release = dnbd3_release,
+	.ioctl = dnbd3_ioctl,
+	.compat_ioctl =	dnbd3_ioctl,
+};
+
+static blk_status_t dnbd3_queue_rq(struct blk_mq_hw_ctx *hctx, const struct blk_mq_queue_data *bd)
+{
+	printk(KERN_DEBUG "dnbd3: dnbd3_queue_rq");
+	return 0;
+}
+
+static void dnbd3_complete_rq(struct request *req)
+{
+	printk(KERN_DEBUG "dnbd3: dnbd3_complete_rq");
+
+}
+
+static int dnbd3_init_request(struct blk_mq_tag_set *set, struct request *rq, unsigned int hctx_idx, unsigned int numa_node)
+{
+	struct dnbd3_cmd *cmd = blk_mq_rq_to_pdu(rq);
+	cmd->dnbd3 = set->driver_data;
+	cmd->flags = 0;
+	mutex_init(&cmd->lock);
+	return 0;
+}
+static enum blk_eh_timer_return dnbd3_xmit_timeout(struct request *req, bool reserved)
+{
+	printk(KERN_DEBUG "dnbd3: dnbd3_xmit_timeout");
+	return BLK_EH_DONE;
+}
+
+
+static const struct blk_mq_ops dnbd3_mq_ops = {
+	.queue_rq = dnbd3_queue_rq,
+	.complete = dnbd3_complete_rq,
+	.init_request = dnbd3_init_request,
+	.timeout = dnbd3_xmit_timeout,
+};
+
+
+static int dnbd3_blk_add_device(dnbd3_device_t *dev, int minor)
+{
+	struct gendisk *disk;
+	struct request_queue *q;
+	int err = -ENOMEM;
+	printk(KERN_DEBUG "dnbd3: adding device %i", minor);
+
+
+	disk = alloc_disk(1);
+	if (!disk) {
+		printk(KERN_DEBUG "dnbd3: alloc_disc failed, device %i", minor);
+		goto out_free_nbd;
+	}
+
+	err = idr_alloc(&dnbd3_index_idr, dev, minor, minor + 1, GFP_KERNEL);
+	if (err == -ENOSPC) {
+		printk(KERN_DEBUG "dnbd3: idr_alloc failed, device %i", minor);
+		err = -EEXIST;
+	}
+
+	if (err < 0)
+		goto out_free_disk;
+
+	dev->minor = minor;
+	dev->disk = disk;
+	dev->tag_set.ops = &dnbd3_mq_ops;
+	dev->tag_set.nr_hw_queues = 1;
+	dev->tag_set.queue_depth = 128;
+	dev->tag_set.numa_node = NUMA_NO_NODE;
+	dev->tag_set.cmd_size = sizeof(dnbd3_cmd);
+	dev->tag_set.flags = BLK_MQ_F_SHOULD_MERGE |
+		BLK_MQ_F_SG_MERGE | BLK_MQ_F_BLOCKING;
+	dev->tag_set.driver_data = dev;
+
+	err = blk_mq_alloc_tag_set(&dev->tag_set);
+	if (err)
+		goto out_free_idr;
+
+	q = blk_mq_init_queue(&dev->tag_set);
+	if (IS_ERR(q)) {
+		err = PTR_ERR(q);
+		goto out_free_tags;
+	}
+	disk->queue = q;
+
+	/*
+	 * Tell the block layer that we are not a rotational device
+	 */
+	blk_queue_flag_set(QUEUE_FLAG_NONROT, disk->queue);
+	blk_queue_flag_clear(QUEUE_FLAG_ADD_RANDOM, disk->queue);
+	disk->queue->limits.discard_granularity = 0;
+	disk->queue->limits.discard_alignment = 0;
+	blk_queue_max_discard_sectors(disk->queue, 0);
+	blk_queue_max_segment_size(disk->queue, UINT_MAX);
+	blk_queue_max_segments(disk->queue, USHRT_MAX);
+	blk_queue_max_hw_sectors(disk->queue, 65536);
+	disk->queue->limits.max_sectors = 256;
+
+	mutex_init(&dev->config_lock);
+	refcount_set(&dev->config_refs, 0);
+	refcount_set(&dev->refs, 1);
+	INIT_LIST_HEAD(&dev->list);
+	disk->major = major;
+	disk->first_minor = minor;
+	disk->fops = &dnbd3_fops;
+	disk->private_data = dev;
+	sprintf(disk->disk_name, "dnbd%d", minor);
+//	sprintf(disk->disk_name, "dnbd3%d", minor);
+	printk(KERN_DEBUG "dnbd3: add disk, device %s", disk->disk_name);
+	add_disk(disk);
+	dnbd3_sysfs_init(dev);
+	return minor;
+
+out_free_tags:
+	blk_mq_free_tag_set(&dev->tag_set);
+out_free_idr:
+	idr_remove(&dnbd3_index_idr, minor);
+out_free_disk:
+	put_disk(disk);
+out_free_nbd:
+	kfree(dev);
+	printk(KERN_DEBUG "dnbd3: destroy device %i", minor);
+	return err;
+}
+
+
+
+static int __init dnbd3_init(void)
+{
+	int i;
+	printk(KERN_DEBUG "dnbd3: starting kernel module");
+
+	if (max_devs < 0) {
+		printk(KERN_ERR "dnbd3: max_devs must be >= 0");
+		return -EINVAL;
+	}
+
+
+	dnbd3_device = kcalloc(max_devs, sizeof(*dnbd3_device), GFP_KERNEL);
+	if (!dnbd3_device) {
+		printk(KERN_ERR "dnbd3: failed to create dnbd3 device");
+		return -ENOMEM;
+	}
+
+	// initialize block device
+	major = register_blkdev(0, "dnbd3");
+	if (major == 0) {
+		printk(KERN_ERR "dnbd3: register_blkdev failed");
+		return -EIO;
+	}
+
+	printk(KERN_DEBUG "dnbd3: kernel module loaded. Machine type: " ENDIAN_MODE);
+
+	// add MAX_NUMBER_DEVICES devices
+	mutex_lock(&dnbd3_index_mutex);
+	for (i = 0; i < max_devs; i++) {
+		dnbd3_blk_add_device(&dnbd3_device[i], i);
+	}
+	mutex_unlock(&dnbd3_index_mutex);
+
+	printk(KERN_INFO "dnbd3: init successful (%i devices).\n", max_devs);
+
+	return 0;
+}
+
+
+static int dnbd3_exit_cb(int id, void *ptr, void *data)
+{
+	struct list_head *list = (struct list_head *)data;
+	struct dnbd3_device_t *dnbd3 = ptr;
+
+	list_add_tail(&dnbd3->list, list);
+	return 0;
+}
+
+static void dnbd3_dev_remove(struct dnbd3_device_t *dnbd3)
+{
+	struct gendisk *disk = dnbd3->disk;
+	struct request_queue *q;
+
+	if (disk) {
+		q = disk->queue;
+		del_gendisk(disk);
+		blk_cleanup_queue(q);
+		blk_mq_free_tag_set(&dnbd3->tag_set);
+		disk->private_data = NULL;
+		put_disk(disk);
+	}
+}
+
+static void dnbd3_put(struct dnbd3_device_t *dnbd3)
+{
+	if (refcount_dec_and_mutex_lock(&dnbd3->refs, &dnbd3_index_mutex)) {
+		idr_remove(&dnbd3_index_idr, dnbd3->minor);
+		mutex_unlock(&dnbd3_index_mutex);
+		dnbd3_dev_remove(dnbd3);
+	}
+}
+
+
+static void __exit dnbd3_exit(void)
+{
+	dnbd3_device_t *dnbd3;
+	LIST_HEAD(del_list);
+	printk(KERN_DEBUG "dnbd3: stopping kernel module");
+
+	mutex_lock(&dnbd3_index_mutex);
+	idr_for_each(&dnbd3_index_idr, &dnbd3_exit_cb, &del_list);
+	mutex_unlock(&dnbd3_index_mutex);
+
+	while (!list_empty(&del_list)) {
+		dnbd3 = list_first_entry(&del_list, struct dnbd3_device_t, list);
+		dnbd3_sysfs_exit(dnbd3);
+		list_del_init(&dnbd3->list);
+		if (refcount_read(&dnbd3->refs) != 1) {
+			printk(KERN_ERR "dnbd3: possibly leaking a device\n");
+		}
+		dnbd3_put(dnbd3);
+	}
+
+	idr_destroy(&dnbd3_index_idr);
+	unregister_blkdev(major, "dnbd3");
+
+	kfree(dnbd3_device);
+
+	printk(KERN_INFO "dnbd3: stopped kernel module");
+}
+
+
+module_init(dnbd3_init);
+module_exit(dnbd3_exit);
+
+MODULE_DESCRIPTION("Distributed Network Block Device 3");
+MODULE_LICENSE("GPL");
+
+module_param(max_devs, int, 0444);
+MODULE_PARM_DESC(max_devs, "number of network block devices to initialize (default: 8)");