From 79bb581eb0373c717d1ce95e817fd7411b07ff92 Mon Sep 17 00:00:00 2001 From: Johann Latocha Date: Fri, 10 Feb 2012 23:47:32 +0100 Subject: [SERVER] Config reload fixed --- src/kernel/blk.c | 5 +- src/kernel/blk.h | 6 ++ src/kernel/dnbd3.h | 3 - src/kernel/net.c | 210 +++++++++++++++++++++++++++------------------------- src/kernel/net.h | 7 ++ src/server/ipc.c | 10 +-- src/server/net.c | 45 +++++++---- src/server/server.c | 6 +- src/server/utils.c | 8 +- src/types.h | 4 + 10 files changed, 169 insertions(+), 135 deletions(-) (limited to 'src') diff --git a/src/kernel/blk.c b/src/kernel/blk.c index 8a76b0d..4d5b4b5 100644 --- a/src/kernel/blk.c +++ b/src/kernel/blk.c @@ -34,6 +34,7 @@ int dnbd3_blk_add_device(dnbd3_device_t *dev, int minor) dev->vid = 0; dev->rid = 0; dev->sock = NULL; + dev->num_servers = 0; dev->thread_send = NULL; dev->thread_receive = NULL; @@ -63,8 +64,6 @@ int dnbd3_blk_add_device(dnbd3_device_t *dev, int minor) queue_flag_set_unlocked(QUEUE_FLAG_NONROT, disk->queue); dev->disk = disk; - dev->hb_request.cmd_type = REQ_TYPE_SPECIAL; - add_disk(disk); // must be last return 0; } @@ -123,7 +122,9 @@ int dnbd3_blk_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd, u break; default: + kfree(msg); return -1; + } kfree(msg); diff --git a/src/kernel/blk.h b/src/kernel/blk.h index bf5ad80..57d9bfa 100644 --- a/src/kernel/blk.h +++ b/src/kernel/blk.h @@ -23,6 +23,12 @@ #include "dnbd3.h" +enum +{ + REQ_GET_SERVERS = 1, + REQ_GET_FILESIZE = 2, +}; + extern struct block_device_operations dnbd3_blk_ops; int dnbd3_blk_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd, unsigned long arg); diff --git a/src/kernel/dnbd3.h b/src/kernel/dnbd3.h index c260e18..b21dba3 100644 --- a/src/kernel/dnbd3.h +++ b/src/kernel/dnbd3.h @@ -44,12 +44,9 @@ typedef struct int rid; struct socket *sock; struct timer_list hb_timer; - struct request hb_request; - int num_servers; struct in_addr servers[MAX_NUMBER_SERVERS]; - // process struct task_struct *thread_send; struct task_struct *thread_receive; diff --git a/src/kernel/net.c b/src/kernel/net.c index 7afac7d..5bd8871 100644 --- a/src/kernel/net.c +++ b/src/kernel/net.c @@ -19,23 +19,25 @@ */ #include "net.h" +#include "blk.h" #include "utils.h" void dnbd3_net_connect(dnbd3_device_t *dev) { struct sockaddr_in sin; - struct msghdr msg; - struct kvec iov; - dnbd3_request_t dnbd3_request; - dnbd3_reply_t dnbd3_reply; - uint64_t filesize; + struct request *req = kmalloc(sizeof(struct request), GFP_ATOMIC); + // do some checks before connecting + if (!req) + { + printk("ERROR: Kmalloc failed.\n"); + return; + } if (!dev->host || !dev->port || (dev->vid == 0)) { printk("ERROR: Host, port or vid not set.\n"); return; } - if (dev->sock) { printk("ERROR: Device %s already connected to %s.\n", dev->disk->disk_name, dev->host); @@ -62,40 +64,6 @@ void dnbd3_net_connect(dnbd3_device_t *dev) return; } - // prepare message - dnbd3_request.cmd = CMD_GET_SIZE; - dnbd3_request.vid = dev->vid; - dnbd3_request.rid = dev->rid; - msg.msg_name = NULL; - msg.msg_namelen = 0; - msg.msg_control = NULL; - msg.msg_controllen = 0; - msg.msg_flags = MSG_WAITALL | MSG_NOSIGNAL; // No SIGPIPE - - // send message - iov.iov_base = &dnbd3_request; - iov.iov_len = sizeof(dnbd3_request); - kernel_sendmsg(dev->sock, &msg, &iov, 1, sizeof(dnbd3_request)); - - // receive replay - iov.iov_base = &dnbd3_reply; - iov.iov_len = sizeof(dnbd3_reply); - kernel_recvmsg(dev->sock, &msg, &iov, 1, sizeof(dnbd3_reply), msg.msg_flags); - if (dnbd3_reply.size <= 0) - { - printk("ERROR: File size returned by server is < 0.\n"); - sock_release(dev->sock); - dev->sock = NULL; - return; - } - - // receive and set filesize - iov.iov_base = &filesize; - iov.iov_len = sizeof(uint64_t); - kernel_recvmsg(dev->sock, &msg, &iov, 1, dnbd3_reply.size, msg.msg_flags); - set_capacity(dev->disk, filesize >> 9); /* 512 Byte blocks */ - printk("INFO: Filesize %s: %llu\n", dev->disk->disk_name, filesize); - // start sending thread dev->thread_send = kthread_create(dnbd3_net_send, dev, dev->disk->disk_name); wake_up_process(dev->thread_send); @@ -104,12 +72,18 @@ void dnbd3_net_connect(dnbd3_device_t *dev) dev->thread_receive = kthread_create(dnbd3_net_receive, dev, dev->disk->disk_name); wake_up_process(dev->thread_receive); - // Add heartbeat timer + // add heartbeat timer init_timer(&dev->hb_timer); dev->hb_timer.data = (unsigned long) dev; dev->hb_timer.function = dnbd3_net_heartbeat; dev->hb_timer.expires = jiffies + HB_INTERVAL; add_timer(&dev->hb_timer); + + // enqueue request to request_queue_send (ask file size) + req->cmd_type = REQ_TYPE_SPECIAL; + req->cmd_flags = REQ_GET_FILESIZE; + list_add(&req->queuelist, &dev->request_queue_send); + wake_up(&dev->process_queue_send); } void dnbd3_net_disconnect(dnbd3_device_t *dev) @@ -124,7 +98,7 @@ void dnbd3_net_disconnect(dnbd3_device_t *dev) kthread_stop(dev->thread_receive); } - // clear sock + // clear socket if (dev->sock) { sock_release(dev->sock); @@ -151,17 +125,13 @@ void dnbd3_net_disconnect(dnbd3_device_t *dev) int dnbd3_net_send(void *data) { dnbd3_device_t *dev = data; - dnbd3_request_t dnbd3_request; struct request *blk_request; + + dnbd3_request_t dnbd3_request; struct msghdr msg; struct kvec iov; - msg.msg_name = NULL; - msg.msg_namelen = 0; - msg.msg_control = NULL; - msg.msg_controllen = 0; - msg.msg_flags = MSG_WAITALL | MSG_NOSIGNAL; // No SIGPIPE - + init_msghdr(msg); set_user_nice(current, -20); while (!kthread_should_stop() || !list_empty(&dev->request_queue_send)) @@ -178,31 +148,42 @@ int dnbd3_net_send(void *data) list_del_init(&blk_request->queuelist); spin_unlock_irq(&dev->blk_lock); + // what to do? switch (blk_request->cmd_type) { - case REQ_TYPE_SPECIAL: - dnbd3_request.cmd = CMD_GET_SERVERS; - break; - case REQ_TYPE_FS: dnbd3_request.cmd = CMD_GET_BLOCK; dnbd3_request.offset = blk_rq_pos(blk_request) << 9; // *512 dnbd3_request.size = blk_rq_bytes(blk_request); // bytes left to complete entire request break; - default: - printk("ERROR: Unknown command\n"); + case REQ_TYPE_SPECIAL: + switch (blk_request->cmd_flags) + { + case REQ_GET_FILESIZE: + dnbd3_request.cmd = CMD_GET_SIZE; + dnbd3_request.vid = dev->vid; + dnbd3_request.rid = dev->rid; + break; + case REQ_GET_SERVERS: + dnbd3_request.cmd = CMD_GET_SERVERS; + break; + } break; + + default: + printk("ERROR: Unknown command (Send)\n"); + continue; } + // send net request memcpy(dnbd3_request.handle, &blk_request, sizeof(blk_request)); iov.iov_base = &dnbd3_request; iov.iov_len = sizeof(dnbd3_request); - - // send net request if (kernel_sendmsg(dev->sock, &msg, &iov, 1, sizeof(dnbd3_request)) <= 0) printk("ERROR: kernel_sendmsg\n"); + // enqueue request to request_queue_receive spin_lock_irq(&dev->blk_lock); list_add_tail(&blk_request->queuelist, &dev->request_queue_receive); spin_unlock_irq(&dev->blk_lock); @@ -214,27 +195,21 @@ int dnbd3_net_send(void *data) int dnbd3_net_receive(void *data) { dnbd3_device_t *dev = data; + struct request *blk_request, *tmp_request, *received_request; + dnbd3_reply_t dnbd3_reply; - struct request *blk_request; struct msghdr msg; struct kvec iov; struct req_iterator iter; struct bio_vec *bvec; + void *kaddr; unsigned long flags; sigset_t blocked, oldset; - struct request *tmp_request, *received_request; - void *kaddr; - unsigned int size; - int i = 0; - //char a[15]; - - msg.msg_name = NULL; - msg.msg_namelen = 0; - msg.msg_control = NULL; - msg.msg_controllen = 0; - msg.msg_flags = MSG_WAITALL | MSG_NOSIGNAL; // No SIGPIPE + unsigned int size, i; + uint64_t filesize; + init_msghdr(msg); set_user_nice(current, -20); while (!kthread_should_stop() || !list_empty(&dev->request_queue_receive)) @@ -256,15 +231,67 @@ int dnbd3_net_receive(void *data) list_for_each_entry_safe(blk_request, tmp_request, &dev->request_queue_receive, queuelist) { if (blk_request != received_request) - continue; + continue; list_del_init(&blk_request->queuelist); break; } spin_unlock_irq(&dev->blk_lock); + // check if server send error + switch (dnbd3_reply.error) + { + case ERROR_SIZE: + printk("ERROR: Requested image does't exist\n"); + kthread_stop(dev->thread_send); + del_timer(&dev->hb_timer); + sock_release(dev->sock); + kfree(blk_request); + dev->sock = NULL; + return -1; + + case ERROR_RELOAD: + blk_request->cmd_type = REQ_TYPE_SPECIAL; + blk_request->cmd_flags = REQ_GET_FILESIZE; + list_add(&blk_request->queuelist, &dev->request_queue_send); + wake_up(&dev->process_queue_send); + continue; + + } + + // what to do? switch (dnbd3_reply.cmd) { + case CMD_GET_BLOCK: + // receive data and answer to block layer + rq_for_each_segment(bvec, blk_request, iter) + { + siginitsetinv(&blocked, sigmask(SIGKILL)); + sigprocmask(SIG_SETMASK, &blocked, &oldset); + + kaddr = kmap(bvec->bv_page) + bvec->bv_offset; + size = bvec->bv_len; + iov.iov_base = kaddr; + iov.iov_len = size; + kernel_recvmsg(dev->sock, &msg, &iov, 1, size, msg.msg_flags); + kunmap(bvec->bv_page); + + sigprocmask(SIG_SETMASK, &oldset, NULL); + } + spin_lock_irqsave(&dev->blk_lock, flags); + __blk_end_request_all(blk_request, 0); + spin_unlock_irqrestore(&dev->blk_lock, flags); + continue; + + case CMD_GET_SIZE: + iov.iov_base = &filesize; + iov.iov_len = sizeof(uint64_t); + kernel_recvmsg(dev->sock, &msg, &iov, 1, dnbd3_reply.size, msg.msg_flags); + set_capacity(dev->disk, filesize >> 9); /* 512 Byte blocks */ + printk("INFO: Filesize %s: %llu\n", dev->disk->disk_name, filesize); + kfree(blk_request); + continue; + case CMD_GET_SERVERS: dev->num_servers = dnbd3_reply.size / sizeof(struct in_addr); size = sizeof(struct in_addr); @@ -273,38 +300,15 @@ int dnbd3_net_receive(void *data) iov.iov_base = &dev->servers[i]; iov.iov_len = size; kernel_recvmsg(dev->sock, &msg, &iov, 1, size, msg.msg_flags); - -// inet_ntoa(dev->servers[i], a); -// printk("INFO: Server[%i] %s\n", i, a); } - break; - - case CMD_GET_BLOCK: - // receive data and answer to block layer - rq_for_each_segment(bvec, blk_request, iter) - { - siginitsetinv(&blocked, sigmask(SIGKILL)); - sigprocmask(SIG_SETMASK, &blocked, &oldset); - - kaddr = kmap(bvec->bv_page) + bvec->bv_offset; - size = bvec->bv_len; - iov.iov_base = kaddr; - iov.iov_len = size; - kernel_recvmsg(dev->sock, &msg, &iov, 1, size, msg.msg_flags); - kunmap(bvec->bv_page); - - sigprocmask(SIG_SETMASK, &oldset, NULL); - } - spin_lock_irqsave(&dev->blk_lock, flags); - __blk_end_request_all(blk_request, 0); - spin_unlock_irqrestore(&dev->blk_lock, flags); - break; + kfree(blk_request); + continue; default: - printk("ERROR: Unknown command\n"); - break; - } + printk("ERROR: Unknown command (Receive)\n"); + continue; + } } return 0; } @@ -312,8 +316,14 @@ int dnbd3_net_receive(void *data) void dnbd3_net_heartbeat(unsigned long arg) { dnbd3_device_t *dev = (dnbd3_device_t *) arg; - list_add(&dev->hb_request.queuelist, &dev->request_queue_send); - wake_up(&dev->process_queue_send); + struct request *req = kmalloc(sizeof(struct request), GFP_ATOMIC); + if (req) + { + req->cmd_type = REQ_TYPE_SPECIAL; + req->cmd_flags = REQ_GET_SERVERS; + list_add(&req->queuelist, &dev->request_queue_send); + wake_up(&dev->process_queue_send); + } dev->hb_timer.expires = jiffies + HB_INTERVAL; add_timer(&dev->hb_timer); } diff --git a/src/kernel/net.h b/src/kernel/net.h index b9b31a4..ac5d8a4 100644 --- a/src/kernel/net.h +++ b/src/kernel/net.h @@ -23,6 +23,13 @@ #include "dnbd3.h" +#define init_msghdr(h)\ + h.msg_name = NULL;\ + h.msg_namelen = 0;\ + h.msg_control = NULL;\ + h.msg_controllen = 0;\ + h.msg_flags = MSG_WAITALL | MSG_NOSIGNAL; + void dnbd3_net_connect(dnbd3_device_t *lo); void dnbd3_net_disconnect(dnbd3_device_t *lo); diff --git a/src/server/ipc.c b/src/server/ipc.c index b5651ac..4adbb7e 100644 --- a/src/server/ipc.c +++ b/src/server/ipc.c @@ -67,8 +67,7 @@ void* dnbd3_ipc_receive() while (1) { - int cmd; - int num = 0; + int cmd, num = 0; char buf[4096]; // Accept connection @@ -114,9 +113,7 @@ void* dnbd3_ipc_receive() break; } - } - close(server_sock); } @@ -131,7 +128,6 @@ void dnbd3_ipc_send(int cmd) perror("ERROR: IPC socket"); exit(EXIT_FAILURE); } - server.sun_family = AF_UNIX; strcpy(server.sun_path, UNIX_SOCKET); @@ -158,15 +154,12 @@ void dnbd3_ipc_send(int cmd) case IPC_INFO: send(client_sock, &cmd, sizeof(int), MSG_WAITALL); recv(client_sock, &num, sizeof(int), MSG_WAITALL); - printf("INFO: Number clients connected: %i\n", num); - for (i = 0; i < num; i++) { if (recv(client_sock, &buf, sizeof(buf), MSG_WAITALL) > 0) printf("INFO: %s", buf); } - break; default: @@ -174,6 +167,5 @@ void dnbd3_ipc_send(int cmd) break; } - close(client_sock); } diff --git a/src/server/net.c b/src/server/net.c index 4f2ddfa..e4be703 100644 --- a/src/server/net.c +++ b/src/server/net.c @@ -41,6 +41,7 @@ void *dnbd3_handle_query(void *dnbd3_client) dnbd3_reply_t reply; int image_file = -1; + dnbd3_image_t *image; struct in_addr server; int i = 0; @@ -48,13 +49,24 @@ void *dnbd3_handle_query(void *dnbd3_client) while (recv(client->sock, &request, sizeof(dnbd3_request_t), MSG_WAITALL) > 0) { reply.cmd = request.cmd; + reply.error = 0; memcpy(reply.handle, request.handle, sizeof(request.handle)); // TODO: lock CMD_GET_SERVERS and CMD_GET_SIZE because of reloading cfg... + // pthread_spin_lock(&_spinlock); + // pthread_spin_unlock(&_spinlock); switch (request.cmd) { case CMD_GET_SERVERS: + if(!client->image) + { // configuration was reloaded, send error + reply.size = 0; + reply.error = ERROR_RELOAD; + send(client->sock, (char *) &reply, sizeof(dnbd3_reply_t), 0); + continue; + } + if (client->image->num_servers < MAX_NUMBER_SERVERS) reply.size = client->image->num_servers * sizeof(struct in_addr); else @@ -67,30 +79,30 @@ void *dnbd3_handle_query(void *dnbd3_client) inet_aton(client->image->servers[i], &server); send(client->sock, (char *) &server, sizeof(struct in_addr), 0); } - break; + continue; case CMD_GET_SIZE: - pthread_spin_lock(&_spinlock); // because of reloading config - dnbd3_image_t *image = dnbd3_get_image(request.vid, request.rid); - pthread_spin_unlock(&_spinlock); - if (image) - { - image_file = open(image->file, O_RDONLY); - reply.size = sizeof(uint64_t); - client->image = image; - } - else - { + image = dnbd3_get_image(request.vid, request.rid); + + if(!image) + { // image not found, send error printf("ERROR: Client requested an unknown image id.\n"); reply.size = 0; + reply.error = ERROR_SIZE; + send(client->sock, (char *) &reply, sizeof(dnbd3_reply_t), 0); + continue; } + + image_file = open(image->file, O_RDONLY); + reply.size = sizeof(uint64_t); + client->image = image; send(client->sock, (char *) &reply, sizeof(dnbd3_reply_t), 0); send(client->sock, &image->filesize, sizeof(uint64_t), 0); - break; + continue; case CMD_GET_BLOCK: if (image_file < 0) - break; + continue; reply.size = request.size; send(client->sock, (char *) &reply, sizeof(dnbd3_reply_t), 0); @@ -98,11 +110,12 @@ void *dnbd3_handle_query(void *dnbd3_client) if (sendfile(client->sock, image_file, (off_t *) &request.offset, request.size) < 0) printf("ERROR: sendfile returned -1\n"); - break; + continue; default: printf("ERROR: Unknown command\n"); - break; + continue; + } } diff --git a/src/server/server.c b/src/server/server.c index a767ed6..786cc40 100644 --- a/src/server/server.c +++ b/src/server/server.c @@ -110,10 +110,8 @@ int main(int argc, char* argv[]) demonize = 0; break; case 'r': - printf("ERROR: Buggy, do not use...\n"); - // FIXME: segfault in dnbd3_handle_query because of free(_images) - // printf("INFO: Reloading configuration file...\n"); - // dnbd3_ipc_send(IPC_RELOAD); + printf("INFO: Reloading configuration file...\n"); + dnbd3_ipc_send(IPC_RELOAD); return EXIT_SUCCESS; case 's': printf("INFO: Stopping running server...\n"); diff --git a/src/server/utils.c b/src/server/utils.c index 3a9215e..3091b2f 100644 --- a/src/server/utils.c +++ b/src/server/utils.c @@ -72,8 +72,14 @@ void dnbd3_load_config(char *file) void dnbd3_reload_config(char* config_file_name) { - free(_images); _num_images = 0; + GSList *iterator = NULL; + for (iterator = _dnbd3_clients; iterator; iterator = iterator->next) + { + dnbd3_client_t *client = iterator->data; + client->image = NULL; + } + free(_images); dnbd3_load_config(config_file_name); } diff --git a/src/types.h b/src/types.h index 8fdfbba..c236db9 100644 --- a/src/types.h +++ b/src/types.h @@ -42,6 +42,9 @@ typedef struct #define CMD_GET_SIZE 2 #define CMD_GET_SERVERS 3 +#define ERROR_SIZE 1 +#define ERROR_RELOAD 2 + #pragma pack(1) typedef struct { @@ -58,6 +61,7 @@ typedef struct typedef struct { uint16_t cmd; // 2byte + uint16_t error; // 2byte uint64_t size; // 8byte char handle[8]; // 8byte } dnbd3_reply_t; -- cgit v1.2.3-55-g7522