From 94704ea51d3b522f7de88080837e4c1acc820816 Mon Sep 17 00:00:00 2001
From: Johann Latocha
Date: Wed, 25 Jan 2012 21:30:58 +0100
Subject: [KERNEL] Multi device support

---
 build.sh               |  6 +++++
 src/config.h           |  2 +-
 src/kernel/blk.c       | 71 +++++++++++++++++++++++++++++++++++++++++-------
 src/kernel/blk.h       |  4 +++
 src/kernel/core.c      | 65 +++++++++++++-------------------------------
 src/kernel/dnbd3.h     | 37 ++++++++++++++-----------
 src/kernel/net.c       | 73 +++++++++++++++++++++++++-------------------------
 src/kernel/net.h       |  2 +-
 src/server/hashtable.c |  1 -
 src/server/server.c    | 11 ++++----
 src/types.h            |  6 +++--
 11 files changed, 160 insertions(+), 118 deletions(-)
 create mode 100755 build.sh

diff --git a/build.sh b/build.sh
new file mode 100755
index 0000000..6726a86
--- /dev/null
+++ b/build.sh
@@ -0,0 +1,6 @@
+#!/bin/sh
+
+mkdir -p build
+cd build/
+cmake ..
+make
diff --git a/src/config.h b/src/config.h
index 2df4dab..2839bb5 100644
--- a/src/config.h
+++ b/src/config.h
@@ -27,6 +27,7 @@
 // block device
 #define KERNEL_SECTOR_SIZE 512
 #define DNBD3_BLOCK_SIZE 4096
+#define MAX_NUMBER_DEVICES 8
 
 // configuration file
 #define DEFAULT_CONFIG_FILE "/etc/dnbd3-server.conf"
@@ -34,5 +35,4 @@
 #define MAX_FILE_NAME 4096
 #define MAX_FILE_ID 8
 
-
 #endif /* CONFIG_H_ */
diff --git a/src/kernel/blk.c b/src/kernel/blk.c
index 8a31841..213ae23 100644
--- a/src/kernel/blk.c
+++ b/src/kernel/blk.c
@@ -21,26 +21,79 @@
 #include "blk.h"
 #include "net.h"
 
+int dnbd3_blk_add_device(struct dnbd3_device *dev, int minor)
+{
+	struct gendisk *disk;
+	struct request_queue *blk_queue;
+
+	init_waitqueue_head(&dev->process_queue_send);
+	init_waitqueue_head(&dev->process_queue_receive);
+	INIT_LIST_HEAD(&dev->request_queue_send);
+	INIT_LIST_HEAD(&dev->request_queue_receive);
+
+	if (!(disk = alloc_disk(1)))
+	{
+		printk("ERROR: dnbd3 alloc_disk failed.\n");
+		return -EIO;
+	}
+
+	disk->major = major;
+	disk->first_minor = minor;
+	sprintf(disk->disk_name, "dnbd%d", minor);
+	set_capacity(disk, 0);
+	set_disk_ro(disk, 1);
+	disk->fops = &dnbd3_blk_ops;
+
+	spin_lock_init(&dev->blk_lock);
+	if ((blk_queue = blk_init_queue(&dnbd3_blk_request, &dev->blk_lock)) == NULL)
+	{
+		printk("ERROR: dnbd3 blk_init_queue failed.\n");
+		return -EIO;
+	}
+
+	blk_queue_logical_block_size(blk_queue, DNBD3_BLOCK_SIZE);
+	disk->queue = blk_queue;
+	disk->private_data = dev;
+	queue_flag_set_unlocked(QUEUE_FLAG_NONROT, disk->queue);
+	dev->disk = disk;
+
+	add_disk(disk); // must be last
+	return 0;
+}
+
+int dnbd3_blk_del_device(struct dnbd3_device *dev)
+{
+	if (dev->sock)
+		sock_release(dev->sock);
+
+	del_gendisk(dev->disk);
+	put_disk(dev->disk);
+	blk_cleanup_queue(dev->disk->queue);
+	return 0;
+}
+
 struct block_device_operations dnbd3_blk_ops =
 { .owner = THIS_MODULE, .ioctl = dnbd3_blk_ioctl, };
 
 int dnbd3_blk_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd,
 		unsigned long arg)
 {
+	struct dnbd3_device *lo = bdev->bd_disk->private_data;
+
 	switch (cmd)
 	{
 	case IOCTL_SET_HOST:
-		_host = (char *) arg;
+		lo->host = (char *) arg;
 		break;
 
 	case IOCTL_SET_PORT:
-		_port = (char *) arg;
+		lo->port = (char *) arg;
 		break;
 	case IOCTL_SET_IMAGE:
-		_image_id = (char *) arg;
+		lo->image_id = (char *) arg;
 		break;
 	case IOCTL_CONNECT:
-		dnbd3_net_connect();
+		dnbd3_net_connect(lo);
 		break;
 	case BLKFLSBUF:
 		break;
@@ -55,12 +108,12 @@ int dnbd3_blk_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd,
 void dnbd3_blk_request(struct request_queue *q)
 {
 	struct request *req;
-
-	if (!_sock)
-		return;
+	struct dnbd3_device *lo;
 
 	while ((req = blk_fetch_request(q)) != NULL)
 	{
+		lo = req->rq_disk->private_data;
+
 		if (req->cmd_type != REQ_TYPE_FS)
 		{
 			__blk_end_request_all(req, 0);
@@ -69,9 +122,9 @@ void dnbd3_blk_request(struct request_queue *q)
 
 		if (rq_data_dir(req) == READ)
 		{
-			list_add_tail(&req->queuelist, &_request_queue_send);
+			list_add_tail(&req->queuelist, &lo->request_queue_send);
 			spin_unlock_irq(q->queue_lock);
-			wake_up(&_process_queue_send);
+			wake_up(&lo->process_queue_send);
 			spin_lock_irq(q->queue_lock);
 		}
 	}
diff --git a/src/kernel/blk.h b/src/kernel/blk.h
index d2e8162..3092b22 100644
--- a/src/kernel/blk.h
+++ b/src/kernel/blk.h
@@ -30,4 +30,8 @@ int dnbd3_blk_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd,
 
 void dnbd3_blk_request(struct request_queue *q);
 
+int dnbd3_blk_add_device(struct dnbd3_device *dev, int minor);
+
+int dnbd3_blk_del_device(struct dnbd3_device *dev);
+
 #endif /* BLK_H_ */
diff --git a/src/kernel/core.c b/src/kernel/core.c
index 3fe88f3..1483704 100644
--- a/src/kernel/core.c
+++ b/src/kernel/core.c
@@ -21,59 +21,27 @@
 #include "dnbd3.h"
 #include "blk.h"
 
-// block
 int major;
-struct gendisk *disk;
-struct request_queue *dnbd3_queue;
-spinlock_t dnbd3_lock;
-
-// network
-char* _host = NULL;
-char* _port = NULL;
-char* _image_id = NULL;
-struct socket *_sock;
-
-// process
-wait_queue_head_t _process_queue_send;
-wait_queue_head_t _process_queue_receive;
-struct list_head _request_queue_send;
-struct list_head _request_queue_receive;
+struct dnbd3_device dnbd3_device[MAX_NUMBER_DEVICES];
 
 static int __init dnbd3_init(void)
 {
-	// initialize queues
-	init_waitqueue_head(&_process_queue_send);
-	init_waitqueue_head(&_process_queue_receive);
-	INIT_LIST_HEAD(&_request_queue_send);
-	INIT_LIST_HEAD(&_request_queue_receive);
+	int i;
 
 	// initialize block device
-	if ((major = register_blkdev(0, "dnbd")) == 0)
+	if ((major = register_blkdev(0, "dnbd3")) == 0)
 	{
 		printk("ERROR: dnbd3 register_blkdev failed.\n");
 		return -EIO;
 	}
-	if (!(disk = alloc_disk(1)))
-	{
-		printk("ERROR: dnbd3 alloc_disk failed.\n");
-		return -EIO;
-	}
-	disk->major = major;
-	disk->first_minor = 0;
-	sprintf(disk->disk_name, "dnbd0");
-	set_capacity(disk, 0);
-	set_disk_ro(disk, 1);
-	disk->fops = &dnbd3_blk_ops;
-	spin_lock_init(&dnbd3_lock);
-	if ((dnbd3_queue = blk_init_queue(&dnbd3_blk_request, &dnbd3_lock)) == NULL)
+	for (i = 0; i < MAX_NUMBER_DEVICES; i++)
 	{
-		printk("ERROR: dnbd3 blk_init_queue failed.\n");
-		return -EIO;
+		if (dnbd3_blk_add_device(&dnbd3_device[i], i) != 0)
+		{
+			printk("ERROR: adding device failed.\n");
+			return -EIO;
+		}
 	}
-	blk_queue_logical_block_size(dnbd3_queue, DNBD3_BLOCK_SIZE);
-	disk->queue = dnbd3_queue;
-
-	add_disk(disk); // must be last
 
 	printk("INFO: dnbd3 init successful.\n");
 	return 0;
@@ -81,15 +49,18 @@ static int __init dnbd3_init(void)
 
 static void __exit dnbd3_exit(void)
 {
-	if (_sock)
-		sock_release(_sock);
-	unregister_blkdev(major, "dnbd");
-	del_gendisk(disk);
-	put_disk(disk);
-	blk_cleanup_queue(dnbd3_queue);
+	int i;
+	for (i = 0; i < MAX_NUMBER_DEVICES; i++)
+	{
+		dnbd3_blk_del_device(&dnbd3_device[i]);
+	}
+
+	unregister_blkdev(major, "dnbd3");
 	printk("INFO: dnbd3 exit.\n");
 }
 
 module_init( dnbd3_init);
 module_exit( dnbd3_exit);
+
+MODULE_DESCRIPTION("Distributed Network Block Device 3");
 MODULE_LICENSE("GPL");
diff --git a/src/kernel/dnbd3.h b/src/kernel/dnbd3.h
index 71be4e8..561c554 100644
--- a/src/kernel/dnbd3.h
+++ b/src/kernel/dnbd3.h
@@ -29,20 +29,27 @@
 #include "config.h"
 #include "types.h"
 
-// block
-extern struct gendisk *disk;
-extern spinlock_t dnbd3_lock;
-
-// network
-extern char* _host;
-extern char* _port;
-extern char* _image_id;
-extern struct socket *_sock;
-
-// process
-extern wait_queue_head_t _process_queue_send;
-extern wait_queue_head_t _process_queue_receive;
-extern struct list_head _request_queue_send;
-extern struct list_head _request_queue_receive;
+extern int major;
+
+struct dnbd3_device
+{
+
+	// block
+	struct gendisk *disk;
+	spinlock_t blk_lock;
+
+	// network
+	char* host;
+	char* port;
+	char* image_id;
+	struct socket *sock;
+
+	// process
+	wait_queue_head_t process_queue_send;
+	wait_queue_head_t process_queue_receive;
+	struct list_head request_queue_send;
+	struct list_head request_queue_receive;
+
+};
 
 #endif /* DNBD_H_ */
diff --git a/src/kernel/net.c b/src/kernel/net.c
index ac8e81d..9a323e8 100644
--- a/src/kernel/net.c
+++ b/src/kernel/net.c
@@ -21,7 +21,7 @@
 #include "net.h"
 #include "utils.h"
 
-void dnbd3_net_connect(void)
+void dnbd3_net_connect(struct dnbd3_device *lo)
 {
 	struct sockaddr_in sin;
 	struct msghdr msg;
@@ -31,23 +31,23 @@ void dnbd3_net_connect(void)
 	struct task_struct *thread_send;
 	struct task_struct *thread_receive;
 
-	if (!_host || !_port || !_image_id)
+	if (!lo->host || !lo->port || !lo->image_id)
 	{
 		printk("ERROR: Host or port not set.");
 		return;
 	}
 
 	// initialize socket
-	if (sock_create_kern(AF_INET, SOCK_STREAM, IPPROTO_TCP, &_sock) < 0)
+	if (sock_create_kern(AF_INET, SOCK_STREAM, IPPROTO_TCP, &lo->sock) < 0)
 	{
 		printk("ERROR: dnbd3 couldn't create socket.\n");
 		return;
 	}
-	_sock->sk->sk_allocation = GFP_NOIO;
+	lo->sock->sk->sk_allocation = GFP_NOIO;
 	sin.sin_family = AF_INET;
-	sin.sin_addr.s_addr = inet_addr(_host);
-	sin.sin_port = htons(simple_strtol(_port, NULL, 10));
-	if (kernel_connect(_sock, (struct sockaddr *) &sin, sizeof(sin), 0) < 0)
+	sin.sin_addr.s_addr = inet_addr(lo->host);
+	sin.sin_port = htons(simple_strtol(lo->port, NULL, 10));
+	if (kernel_connect(lo->sock, (struct sockaddr *) &sin, sizeof(sin), 0) < 0)
 	{
 		printk("ERROR: dnbd3 couldn't connect to given host.\n");
 		return;
@@ -55,7 +55,7 @@ void dnbd3_net_connect(void)
 
 	// prepare message and send request
 	dnbd3_request.cmd = CMD_GET_SIZE;
-	strcpy(dnbd3_request.image_id, _image_id);
+	strcpy(dnbd3_request.image_id, lo->image_id);
 	msg.msg_name = NULL;
 	msg.msg_namelen = 0;
 	msg.msg_control = NULL;
@@ -64,12 +64,12 @@ void dnbd3_net_connect(void)
 
 	iov.iov_base = &dnbd3_request;
 	iov.iov_len = sizeof(dnbd3_request);
-	kernel_sendmsg(_sock, &msg, &iov, 1, sizeof(dnbd3_request));
+	kernel_sendmsg(lo->sock, &msg, &iov, 1, sizeof(dnbd3_request));
 
 	// receive replay
 	iov.iov_base = &dnbd3_reply;
 	iov.iov_len = sizeof(dnbd3_reply);
-	kernel_recvmsg(_sock, &msg, &iov, 1, sizeof(dnbd3_reply), msg.msg_flags);
+	kernel_recvmsg(lo->sock, &msg, &iov, 1, sizeof(dnbd3_reply), msg.msg_flags);
 
 	// set filesize
 	if (dnbd3_reply.filesize <= 0)
@@ -79,19 +79,20 @@ void dnbd3_net_connect(void)
 	}
 
 	printk("INFO: dnbd3 filesize: %llu\n", dnbd3_reply.filesize);
-	set_capacity(disk, dnbd3_reply.filesize >> 9); /* 512 Byte blocks */
+	set_capacity(lo->disk, dnbd3_reply.filesize >> 9); /* 512 Byte blocks */
 
 	// start sending thread
-	thread_send = kthread_create(dnbd3_net_send, NULL, "none");
+	thread_send = kthread_create(dnbd3_net_send, lo, lo->disk->disk_name);
 	wake_up_process(thread_send);
 
 	// start receiving thread
-	thread_receive = kthread_create(dnbd3_net_receive, NULL, "none");
+	thread_receive = kthread_create(dnbd3_net_receive, lo, lo->disk->disk_name);
 	wake_up_process(thread_receive);
 }
 
 int dnbd3_net_send(void *data)
 {
+	struct dnbd3_device *lo = data;
 	struct dnbd3_request dnbd3_request;
 	struct request *blk_request;
 	struct msghdr msg;
@@ -105,19 +106,19 @@ int dnbd3_net_send(void *data)
 
 	set_user_nice(current, -20);
 
-	while (!kthread_should_stop() || !list_empty(&_request_queue_send))
+	while (!kthread_should_stop() || !list_empty(&lo->request_queue_send))
 	{
-		wait_event_interruptible(_process_queue_send,
-				kthread_should_stop() || !list_empty(&_request_queue_send));
+		wait_event_interruptible(lo->process_queue_send,
+				kthread_should_stop() || !list_empty(&lo->request_queue_send));
 
-		if (list_empty(&_request_queue_send))
+		if (list_empty(&lo->request_queue_send))
 			continue;
 
 		// extract block request
-		spin_lock_irq(&dnbd3_lock);
-		blk_request = list_entry(_request_queue_send.next, struct request, queuelist);
+		spin_lock_irq(&lo->blk_lock);
+		blk_request = list_entry(lo->request_queue_send.next, struct request, queuelist);
 		list_del_init(&blk_request->queuelist);
-		spin_unlock_irq(&dnbd3_lock);
+		spin_unlock_irq(&lo->blk_lock);
 
 		// prepare net request
 		dnbd3_request.cmd = CMD_GET_BLOCK;
@@ -128,19 +129,20 @@ int dnbd3_net_send(void *data)
 		iov.iov_len = sizeof(dnbd3_request);
 
 		// send net request
-		if (kernel_sendmsg(_sock, &msg, &iov, 1, sizeof(dnbd3_request)) <= 0)
+		if (kernel_sendmsg(lo->sock, &msg, &iov, 1, sizeof(dnbd3_request)) <= 0)
 			printk("ERROR: kernel_sendmsg\n");
 
-		spin_lock_irq(&dnbd3_lock);
-		list_add_tail(&blk_request->queuelist, &_request_queue_receive);
-		spin_unlock_irq(&dnbd3_lock);
-		wake_up(&_process_queue_receive);
+		spin_lock_irq(&lo->blk_lock);
+		list_add_tail(&blk_request->queuelist, &lo->request_queue_receive);
+		spin_unlock_irq(&lo->blk_lock);
+		wake_up(&lo->process_queue_receive);
 	}
 	return 0;
 }
 
 int dnbd3_net_receive(void *data)
 {
+	struct dnbd3_device *lo = data;
 	struct dnbd3_reply dnbd3_reply;
 	struct request *blk_request;
 	struct msghdr msg;
@@ -161,22 +163,20 @@ int dnbd3_net_receive(void *data)
 
 	set_user_nice(current, -20);
 
-	while (!kthread_should_stop() || !list_empty(&_request_queue_receive))
+	while (!kthread_should_stop() || !list_empty(&lo->request_queue_receive))
 	{
-		wait_event_interruptible(_process_queue_receive,
-				kthread_should_stop() || !list_empty(&_request_queue_receive));
+		wait_event_interruptible(lo->process_queue_receive, kthread_should_stop() || !list_empty(&lo->request_queue_receive));
 
 		// receive net replay
 		iov.iov_base = &dnbd3_reply;
 		iov.iov_len = sizeof(dnbd3_reply);
-		kernel_recvmsg(_sock, &msg, &iov, 1, sizeof(dnbd3_reply),
+		kernel_recvmsg(lo->sock, &msg, &iov, 1, sizeof(dnbd3_reply),
 				msg.msg_flags);
 
 		// search for replied request in queue
 		received_request = *(struct request **) dnbd3_reply.handle;
-		spin_lock_irq(&dnbd3_lock);
-		list_for_each_entry_safe(blk_request, tmp_request,
-				&_request_queue_receive, queuelist)
+		spin_lock_irq(&lo->blk_lock);
+		list_for_each_entry_safe(blk_request, tmp_request, &lo->request_queue_receive, queuelist)
 		{
 			if (blk_request != received_request)
 				continue;
@@ -184,7 +184,7 @@ int dnbd3_net_receive(void *data)
 			list_del_init(&blk_request->queuelist);
 			break;
 		}
-		spin_unlock_irq(&dnbd3_lock);
+		spin_unlock_irq(&lo->blk_lock);
 
 		// receive data and answer to block layer
 		rq_for_each_segment(bvec, blk_request, iter)
@@ -196,17 +196,16 @@ int dnbd3_net_receive(void *data)
 				size = bvec->bv_len;
 				iov.iov_base = kaddr;
 				iov.iov_len = size;
-				kernel_recvmsg(_sock, &msg, &iov, 1, size, msg.msg_flags);
+				kernel_recvmsg(lo->sock, &msg, &iov, 1, size, msg.msg_flags);
 				kunmap(bvec->bv_page);
 
 				sigprocmask(SIG_SETMASK, &oldset, NULL);
 			}
 
-		spin_lock_irqsave(&dnbd3_lock, flags);
+		spin_lock_irqsave(&lo->blk_lock, flags);
 		__blk_end_request_all(blk_request, 0);
-		spin_unlock_irqrestore(&dnbd3_lock, flags);
+		spin_unlock_irqrestore(&lo->blk_lock, flags);
 	}
 
 	return 0;
 }
-
diff --git a/src/kernel/net.h b/src/kernel/net.h
index 22c0cbd..5ac8f6f 100644
--- a/src/kernel/net.h
+++ b/src/kernel/net.h
@@ -23,7 +23,7 @@
 
 #include "dnbd3.h"
 
-void dnbd3_net_connect(void);
+void dnbd3_net_connect(struct dnbd3_device *lo);
 
 int dnbd3_net_send(void *data);
 
diff --git a/src/server/hashtable.c b/src/server/hashtable.c
index 518454f..2099a65 100644
--- a/src/server/hashtable.c
+++ b/src/server/hashtable.c
@@ -67,4 +67,3 @@ char* ht_search(char* key)
 	else
 		return NULL;
 }
-
diff --git a/src/server/server.c b/src/server/server.c
index 7185e2f..c266f19 100644
--- a/src/server/server.c
+++ b/src/server/server.c
@@ -105,7 +105,8 @@ void *handle_query(void *client_socket)
 			memcpy(reply.handle, request.handle, sizeof(request.handle));
 			send(sock, (char *) &reply, sizeof(struct dnbd3_reply), 0);
 
-			if (sendfile(sock, image_file, (off_t *) &request.offset, request.size) <0)
+			if (sendfile(sock, image_file, (off_t *) &request.offset,
+					request.size) < 0)
 				printf("ERROR: sendfile returned -1\n");
 
 			break;
@@ -118,7 +119,7 @@ void *handle_query(void *client_socket)
 	}
 	close(sock);
 	printf("Client exit.\n");
-	pthread_exit((void *)0);
+	pthread_exit((void *) 0);
 }
 
 int main(int argc, char* argv[])
@@ -157,7 +158,7 @@ int main(int argc, char* argv[])
 
 	// parse config file
 	ht_create();
-	FILE *config_file = fopen(config_file_name , "r");
+	FILE *config_file = fopen(config_file_name, "r");
 	if (config_file == NULL)
 	{
 		printf("ERROR: Config file not found: %s\n", config_file_name);
@@ -166,9 +167,9 @@ int main(int argc, char* argv[])
 	char line[MAX_FILE_NAME + 1 + MAX_FILE_ID];
 	char* image_name = NULL;
 	char* image_id = NULL;
-	while (fgets (line , sizeof(line) , config_file) != NULL )
+	while (fgets(line, sizeof(line), config_file) != NULL)
 	{
-		sscanf (line, "%as %as", &image_name, &image_id);
+		sscanf(line, "%as %as", &image_name, &image_id);
 		if (ht_insert(image_id, image_name) < 0)
 		{
 			printf("ERROR: Image name or ID is too big\n");
diff --git a/src/types.h b/src/types.h
index f3b2b97..f4ff98e 100644
--- a/src/types.h
+++ b/src/types.h
@@ -35,7 +35,8 @@
 #define CMD_GET_SIZE    2
 
 #pragma pack(1)
-typedef struct dnbd3_request {
+typedef struct dnbd3_request
+{
 	uint16_t cmd;
 	uint64_t offset;
 	uint64_t size;
@@ -45,7 +46,8 @@ typedef struct dnbd3_request {
 #pragma pack(0)
 
 #pragma pack(1)
-typedef struct dnbd3_reply {
+typedef struct dnbd3_reply
+{
 	uint16_t cmd;
 	uint64_t filesize;
 	char handle[8];
-- 
cgit v1.2.3-55-g7522