From f7e732e0ab494627666f3affe23622729bd637d7 Mon Sep 17 00:00:00 2001 From: Johann Latocha Date: Wed, 8 Feb 2012 21:26:47 +0100 Subject: [ALL] Refactoring [ALL] Send and receive servers [TODO] Config file reloading is broken --- src/kernel/blk.c | 34 +++++----- src/kernel/dnbd3.h | 4 ++ src/kernel/net.c | 180 +++++++++++++++++++++++++++++------------------------ src/kernel/utils.c | 8 +++ src/kernel/utils.h | 3 + 5 files changed, 130 insertions(+), 99 deletions(-) (limited to 'src/kernel') diff --git a/src/kernel/blk.c b/src/kernel/blk.c index 009e3d4..8a76b0d 100644 --- a/src/kernel/blk.c +++ b/src/kernel/blk.c @@ -91,8 +91,8 @@ struct block_device_operations dnbd3_blk_ops = int dnbd3_blk_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd, unsigned long arg) { - dnbd3_device_t *lo = bdev->bd_disk->private_data; - int minor = lo->disk->first_minor; + dnbd3_device_t *dev = bdev->bd_disk->private_data; + int minor = dev->disk->first_minor; dnbd3_ioctl_t *msg = kmalloc(sizeof(dnbd3_ioctl_t), GFP_KERNEL); copy_from_user((char *)msg, (char *)arg, sizeof(*msg)); @@ -100,23 +100,23 @@ int dnbd3_blk_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd, u switch (cmd) { case IOCTL_OPEN: - strcpy(lo->host, msg->host); - strcpy(lo->port, msg->port); - lo->vid = msg->vid; - lo->rid = msg->rid; - dnbd3_net_connect(lo); + strcpy(dev->host, msg->host); + strcpy(dev->port, msg->port); + dev->vid = msg->vid; + dev->rid = msg->rid; + dnbd3_net_connect(dev); break; case IOCTL_CLOSE: - dnbd3_net_disconnect(lo); - dnbd3_blk_del_device(lo); - dnbd3_blk_add_device(lo, minor); + dnbd3_net_disconnect(dev); + dnbd3_blk_del_device(dev); + dnbd3_blk_add_device(dev, minor); break; case IOCTL_SWITCH: - dnbd3_net_disconnect(lo); - strcpy(lo->host, msg->host); - dnbd3_net_connect(lo); + dnbd3_net_disconnect(dev); + strcpy(dev->host, msg->host); + dnbd3_net_connect(dev); break; case BLKFLSBUF: @@ -133,11 +133,11 @@ int dnbd3_blk_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd, u void dnbd3_blk_request(struct request_queue *q) { struct request *req; - dnbd3_device_t *lo; + dnbd3_device_t *dev; while ((req = blk_fetch_request(q)) != NULL) { - lo = req->rq_disk->private_data; + dev = req->rq_disk->private_data; if (req->cmd_type != REQ_TYPE_FS) { @@ -147,9 +147,9 @@ void dnbd3_blk_request(struct request_queue *q) if (rq_data_dir(req) == READ) { - list_add_tail(&req->queuelist, &lo->request_queue_send); + list_add_tail(&req->queuelist, &dev->request_queue_send); spin_unlock_irq(q->queue_lock); - wake_up(&lo->process_queue_send); + wake_up(&dev->process_queue_send); spin_lock_irq(q->queue_lock); } } diff --git a/src/kernel/dnbd3.h b/src/kernel/dnbd3.h index 8dc0f47..c260e18 100644 --- a/src/kernel/dnbd3.h +++ b/src/kernel/dnbd3.h @@ -46,6 +46,10 @@ typedef struct struct timer_list hb_timer; struct request hb_request; + int num_servers; + struct in_addr servers[MAX_NUMBER_SERVERS]; + + // process struct task_struct *thread_send; struct task_struct *thread_receive; diff --git a/src/kernel/net.c b/src/kernel/net.c index a90dfb8..11edd0c 100644 --- a/src/kernel/net.c +++ b/src/kernel/net.c @@ -21,50 +21,51 @@ #include "net.h" #include "utils.h" -void dnbd3_net_connect(dnbd3_device_t *lo) +void dnbd3_net_connect(dnbd3_device_t *dev) { struct sockaddr_in sin; struct msghdr msg; struct kvec iov; dnbd3_request_t dnbd3_request; dnbd3_reply_t dnbd3_reply; + uint64_t filesize; - if (!lo->host || !lo->port || (lo->vid == 0)) + if (!dev->host || !dev->port || (dev->vid == 0)) { printk("ERROR: Host, port or vid not set.\n"); return; } - if (lo->sock) + if (dev->sock) { - printk("ERROR: Device %s already connected to %s.\n", lo->disk->disk_name, lo->host); + printk("ERROR: Device %s already connected to %s.\n", dev->disk->disk_name, dev->host); return; } - printk("INFO: Connecting device %s to %s\n", lo->disk->disk_name, lo->host); + printk("INFO: Connecting device %s to %s\n", dev->disk->disk_name, dev->host); // initialize socket - if (sock_create_kern(AF_INET, SOCK_STREAM, IPPROTO_TCP, &lo->sock) < 0) + if (sock_create_kern(AF_INET, SOCK_STREAM, IPPROTO_TCP, &dev->sock) < 0) { printk("ERROR: Couldn't create socket.\n"); - lo->sock = NULL; + dev->sock = NULL; return; } - lo->sock->sk->sk_allocation = GFP_NOIO; + dev->sock->sk->sk_allocation = GFP_NOIO; sin.sin_family = AF_INET; - sin.sin_addr.s_addr = inet_addr(lo->host); - sin.sin_port = htons(simple_strtol(lo->port, NULL, 10)); - if (kernel_connect(lo->sock, (struct sockaddr *) &sin, sizeof(sin), 0) < 0) + sin.sin_addr.s_addr = inet_addr(dev->host); + sin.sin_port = htons(simple_strtol(dev->port, NULL, 10)); + if (kernel_connect(dev->sock, (struct sockaddr *) &sin, sizeof(sin), 0) < 0) { - printk("ERROR: Couldn't connect to host %s:%s\n", lo->host, lo->port); - lo->sock = NULL; + printk("ERROR: Couldn't connect to host %s:%s\n", dev->host, dev->port); + dev->sock = NULL; return; } // prepare message dnbd3_request.cmd = CMD_GET_SIZE; - dnbd3_request.vid = lo->vid; - dnbd3_request.rid = lo->rid; + dnbd3_request.vid = dev->vid; + dnbd3_request.rid = dev->rid; msg.msg_name = NULL; msg.msg_namelen = 0; msg.msg_control = NULL; @@ -74,80 +75,82 @@ void dnbd3_net_connect(dnbd3_device_t *lo) // send message iov.iov_base = &dnbd3_request; iov.iov_len = sizeof(dnbd3_request); - kernel_sendmsg(lo->sock, &msg, &iov, 1, sizeof(dnbd3_request)); + kernel_sendmsg(dev->sock, &msg, &iov, 1, sizeof(dnbd3_request)); // receive replay iov.iov_base = &dnbd3_reply; iov.iov_len = sizeof(dnbd3_reply); - kernel_recvmsg(lo->sock, &msg, &iov, 1, sizeof(dnbd3_reply), msg.msg_flags); - - // set filesize - if (dnbd3_reply.filesize <= 0) + kernel_recvmsg(dev->sock, &msg, &iov, 1, sizeof(dnbd3_reply), msg.msg_flags); + if (dnbd3_reply.size <= 0) { printk("ERROR: File size returned by server is < 0.\n"); - sock_release(lo->sock); - lo->sock = NULL; + sock_release(dev->sock); + dev->sock = NULL; return; } - set_capacity(lo->disk, dnbd3_reply.filesize >> 9); /* 512 Byte blocks */ - printk("INFO: Filesize %s: %llu\n", lo->disk->disk_name, dnbd3_reply.filesize); + // receive and set filesize + iov.iov_base = &filesize; + iov.iov_len = sizeof(uint64_t); + kernel_recvmsg(dev->sock, &msg, &iov, 1, dnbd3_reply.size, msg.msg_flags); + set_capacity(dev->disk, filesize >> 9); /* 512 Byte blocks */ + printk("INFO: Filesize %s: %llu\n", dev->disk->disk_name, filesize); // start sending thread - lo->thread_send = kthread_create(dnbd3_net_send, lo, lo->disk->disk_name); - wake_up_process(lo->thread_send); + dev->thread_send = kthread_create(dnbd3_net_send, dev, dev->disk->disk_name); + wake_up_process(dev->thread_send); // start receiving thread - lo->thread_receive = kthread_create(dnbd3_net_receive, lo, lo->disk->disk_name); - wake_up_process(lo->thread_receive); + dev->thread_receive = kthread_create(dnbd3_net_receive, dev, dev->disk->disk_name); + wake_up_process(dev->thread_receive); // Add heartbeat timer - init_timer(&lo->hb_timer); - lo->hb_timer.data = (unsigned long) lo; - lo->hb_timer.function = dnbd3_net_heartbeat; - lo->hb_timer.expires = jiffies + HB_INTERVAL; - add_timer(&lo->hb_timer); + init_timer(&dev->hb_timer); + dev->hb_timer.data = (unsigned long) dev; + dev->hb_timer.function = dnbd3_net_heartbeat; + dev->hb_timer.expires = jiffies + HB_INTERVAL; + add_timer(&dev->hb_timer); } -void dnbd3_net_disconnect(dnbd3_device_t *lo) +void dnbd3_net_disconnect(dnbd3_device_t *dev) { struct request *blk_request, *tmp_request; - printk("INFO: Disconnecting device %s\n", lo->disk->disk_name); + printk("INFO: Disconnecting device %s\n", dev->disk->disk_name); // kill sending and receiving threads - if (lo->thread_send && lo->thread_receive) + if (dev->thread_send && dev->thread_receive) { - kthread_stop(lo->thread_send); - kthread_stop(lo->thread_receive); + kthread_stop(dev->thread_send); + kthread_stop(dev->thread_receive); } // clear sock - if (lo->sock) + if (dev->sock) { - sock_release(lo->sock); - lo->sock = NULL; + sock_release(dev->sock); + dev->sock = NULL; } // clear heartbeat timer - if (&lo->hb_timer) - del_timer(&lo->hb_timer); + if (&dev->hb_timer) + del_timer(&dev->hb_timer); // move already send requests to request_queue_send again - if (!list_empty(&lo->request_queue_receive)) + if (!list_empty(&dev->request_queue_receive)) { - printk("WARN: Request queue was not empty on %s\n", lo->disk->disk_name); - spin_lock_irq(&lo->blk_lock); - list_for_each_entry_safe(blk_request, tmp_request, &lo->request_queue_receive, queuelist) + printk("WARN: Request queue was not empty on %s\n", dev->disk->disk_name); + spin_lock_irq(&dev->blk_lock); + list_for_each_entry_safe(blk_request, tmp_request, &dev->request_queue_receive, queuelist) { list_del_init(&blk_request->queuelist); - list_add_tail(&blk_request->queuelist, &lo->request_queue_send); + list_add_tail(&blk_request->queuelist, &dev->request_queue_send); } - spin_unlock_irq(&lo->blk_lock); + spin_unlock_irq(&dev->blk_lock); } } int dnbd3_net_send(void *data) { - dnbd3_device_t *lo = data; + dnbd3_device_t *dev = data; dnbd3_request_t dnbd3_request; struct request *blk_request; struct msghdr msg; @@ -161,24 +164,24 @@ int dnbd3_net_send(void *data) set_user_nice(current, -20); - while (!kthread_should_stop() || !list_empty(&lo->request_queue_send)) + while (!kthread_should_stop() || !list_empty(&dev->request_queue_send)) { - wait_event_interruptible(lo->process_queue_send, - kthread_should_stop() || !list_empty(&lo->request_queue_send)); + wait_event_interruptible(dev->process_queue_send, + kthread_should_stop() || !list_empty(&dev->request_queue_send)); - if (list_empty(&lo->request_queue_send)) + if (list_empty(&dev->request_queue_send)) continue; // extract block request - spin_lock_irq(&lo->blk_lock); - blk_request = list_entry(lo->request_queue_send.next, struct request, queuelist); + spin_lock_irq(&dev->blk_lock); + blk_request = list_entry(dev->request_queue_send.next, struct request, queuelist); list_del_init(&blk_request->queuelist); - spin_unlock_irq(&lo->blk_lock); + spin_unlock_irq(&dev->blk_lock); switch (blk_request->cmd_type) { case REQ_TYPE_SPECIAL: - dnbd3_request.cmd = CMD_PING; + dnbd3_request.cmd = CMD_GET_SERVERS; break; case REQ_TYPE_FS: @@ -197,20 +200,20 @@ int dnbd3_net_send(void *data) iov.iov_len = sizeof(dnbd3_request); // send net request - if (kernel_sendmsg(lo->sock, &msg, &iov, 1, sizeof(dnbd3_request)) <= 0) + if (kernel_sendmsg(dev->sock, &msg, &iov, 1, sizeof(dnbd3_request)) <= 0) printk("ERROR: kernel_sendmsg\n"); - spin_lock_irq(&lo->blk_lock); - list_add_tail(&blk_request->queuelist, &lo->request_queue_receive); - spin_unlock_irq(&lo->blk_lock); - wake_up(&lo->process_queue_receive); + spin_lock_irq(&dev->blk_lock); + list_add_tail(&blk_request->queuelist, &dev->request_queue_receive); + spin_unlock_irq(&dev->blk_lock); + wake_up(&dev->process_queue_receive); } return 0; } int dnbd3_net_receive(void *data) { - dnbd3_device_t *lo = data; + dnbd3_device_t *dev = data; dnbd3_reply_t dnbd3_reply; struct request *blk_request; struct msghdr msg; @@ -223,6 +226,9 @@ int dnbd3_net_receive(void *data) void *kaddr; unsigned int size; + int i = 0; + char a[15]; + msg.msg_name = NULL; msg.msg_namelen = 0; msg.msg_control = NULL; @@ -231,23 +237,23 @@ int dnbd3_net_receive(void *data) set_user_nice(current, -20); - while (!kthread_should_stop() || !list_empty(&lo->request_queue_receive)) + while (!kthread_should_stop() || !list_empty(&dev->request_queue_receive)) { - wait_event_interruptible(lo->process_queue_receive, - kthread_should_stop() || !list_empty(&lo->request_queue_receive)); + wait_event_interruptible(dev->process_queue_receive, + kthread_should_stop() || !list_empty(&dev->request_queue_receive)); - if (list_empty(&lo->request_queue_receive)) + if (list_empty(&dev->request_queue_receive)) continue; // receive net replay iov.iov_base = &dnbd3_reply; iov.iov_len = sizeof(dnbd3_reply); - kernel_recvmsg(lo->sock, &msg, &iov, 1, sizeof(dnbd3_reply), msg.msg_flags); + kernel_recvmsg(dev->sock, &msg, &iov, 1, sizeof(dnbd3_reply), msg.msg_flags); // search for replied request in queue received_request = *(struct request **) dnbd3_reply.handle; - spin_lock_irq(&lo->blk_lock); - list_for_each_entry_safe(blk_request, tmp_request, &lo->request_queue_receive, queuelist) + spin_lock_irq(&dev->blk_lock); + list_for_each_entry_safe(blk_request, tmp_request, &dev->request_queue_receive, queuelist) { if (blk_request != received_request) continue; @@ -255,12 +261,22 @@ int dnbd3_net_receive(void *data) list_del_init(&blk_request->queuelist); break; } - spin_unlock_irq(&lo->blk_lock); + spin_unlock_irq(&dev->blk_lock); switch (dnbd3_reply.cmd) { - case CMD_PING: - // TODO: use for rtt? + case CMD_GET_SERVERS: + dev->num_servers = dnbd3_reply.size / sizeof(struct in_addr); + size = sizeof(struct in_addr); + for (i = 0; i < dev->num_servers && i < MAX_NUMBER_SERVERS; i++) + { + iov.iov_base = &dev->servers[i]; + iov.iov_len = size; + kernel_recvmsg(dev->sock, &msg, &iov, 1, size, msg.msg_flags); + +// inet_ntoa(dev->servers[i], a); +// printk("INFO: Server[%i] %s\n", i, a); + } break; case CMD_GET_BLOCK: @@ -274,14 +290,14 @@ int dnbd3_net_receive(void *data) size = bvec->bv_len; iov.iov_base = kaddr; iov.iov_len = size; - kernel_recvmsg(lo->sock, &msg, &iov, 1, size, msg.msg_flags); + kernel_recvmsg(dev->sock, &msg, &iov, 1, size, msg.msg_flags); kunmap(bvec->bv_page); sigprocmask(SIG_SETMASK, &oldset, NULL); } - spin_lock_irqsave(&lo->blk_lock, flags); + spin_lock_irqsave(&dev->blk_lock, flags); __blk_end_request_all(blk_request, 0); - spin_unlock_irqrestore(&lo->blk_lock, flags); + spin_unlock_irqrestore(&dev->blk_lock, flags); break; default: @@ -295,9 +311,9 @@ int dnbd3_net_receive(void *data) void dnbd3_net_heartbeat(unsigned long arg) { - dnbd3_device_t *lo = (dnbd3_device_t *) arg; - list_add(&lo->hb_request.queuelist, &lo->request_queue_send); - wake_up(&lo->process_queue_send); - lo->hb_timer.expires = jiffies + HB_INTERVAL; - add_timer(&lo->hb_timer); + dnbd3_device_t *dev = (dnbd3_device_t *) arg; + list_add(&dev->hb_request.queuelist, &dev->request_queue_send); + wake_up(&dev->process_queue_send); + dev->hb_timer.expires = jiffies + HB_INTERVAL; + add_timer(&dev->hb_timer); } diff --git a/src/kernel/utils.c b/src/kernel/utils.c index 85c0ce0..221dc4d 100644 --- a/src/kernel/utils.c +++ b/src/kernel/utils.c @@ -20,6 +20,8 @@ #include +#include "utils.h" + unsigned int inet_addr(char *str) { int a, b, c, d; @@ -31,3 +33,9 @@ unsigned int inet_addr(char *str) arr[3] = d; return *(unsigned int*) arr; } + +void inet_ntoa(struct in_addr addr, char* str) +{ + unsigned char *ptr = (unsigned char *) &addr; + sprintf(str, "%d.%d.%d.%d", ptr[0] & 0xff, ptr[1] & 0xff, ptr[2] & 0xff, ptr[3] & 0xff); +} diff --git a/src/kernel/utils.h b/src/kernel/utils.h index 6b7fcee..b2c5fb1 100644 --- a/src/kernel/utils.h +++ b/src/kernel/utils.h @@ -21,6 +21,9 @@ #ifndef UTILS_H_ #define UTILS_H_ +#include + unsigned int inet_addr(char *str); +void inet_ntoa(struct in_addr addr, char* str); #endif /* UTILS_H_ */ -- cgit v1.2.3-55-g7522