From 79ae416cbf5618328742234c33bb24fb0a61fe87 Mon Sep 17 00:00:00 2001 From: Simon Rettberg Date: Mon, 11 Nov 2013 22:44:00 +0100 Subject: [KERNEL] Fix possible deadlock on server switch [KERNEL] Remove server mode support as it's not needed anymore [KERNEL] Some more sanity checks and debug messages --- src/kernel/dnbd3.h | 4 +- src/kernel/net.c | 151 ++++++++++++++++++++++++++--------------------------- 2 files changed, 75 insertions(+), 80 deletions(-) diff --git a/src/kernel/dnbd3.h b/src/kernel/dnbd3.h index aa9ea86..c703019 100644 --- a/src/kernel/dnbd3.h +++ b/src/kernel/dnbd3.h @@ -68,8 +68,8 @@ typedef struct struct socket *better_sock; // process - struct task_struct *thread_send; - struct task_struct *thread_receive; + struct task_struct * volatile thread_send; + struct task_struct * volatile thread_receive; struct task_struct *thread_discover; struct timer_list hb_timer; wait_queue_head_t process_queue_send; diff --git a/src/kernel/net.c b/src/kernel/net.c index 8703d9c..9811a31 100644 --- a/src/kernel/net.c +++ b/src/kernel/net.c @@ -122,11 +122,19 @@ int dnbd3_net_connect(dnbd3_device_t *dev) struct request *req1 = NULL; struct timeval timeout; - char get_servers = 0, set_client = 0; - - while (dev->disconnecting) - { - if (dev->better_sock) + if (dev->disconnecting) { + debug_dev("CONNECT: Still disconnecting!!!\n"); + while (dev->disconnecting) + schedule(); + } + if (dev->thread_receive != NULL) { + debug_dev("CONNECT: Still receiving!!!\n"); + while (dev->thread_receive != NULL) + schedule(); + } + if (dev->thread_send != NULL) { + debug_dev("CONNECT: Still sending!!!\n"); + while (dev->thread_send != NULL) schedule(); } @@ -135,24 +143,9 @@ int dnbd3_net_connect(dnbd3_device_t *dev) // do some checks before connecting - if (!dev->is_server && is_same_server(&dev->cur_server, &dev->initial_server)) - { - // Forget all known alt servers - memset(dev->alt_servers, 0, sizeof(dev->alt_servers[0]) * NUMBER_SERVERS); - memcpy(dev->alt_servers, &dev->initial_server, sizeof(dev->alt_servers[0])); - get_servers = 1; - } - if (dev->better_sock) - { - set_client = 1; - } - - if (get_servers || set_client) - { - req1 = kmalloc(sizeof(*req1), GFP_ATOMIC ); - if (!req1) - error_dev("FATAL: Kmalloc(1) failed."); - } + req1 = kmalloc(sizeof(*req1), GFP_ATOMIC ); + if (!req1) + error_dev("FATAL: Kmalloc(1) failed."); if (dev->cur_server.host.port == 0 || dev->cur_server.host.type == 0 || dev->imgname == NULL ) error_dev("FATAL: Host, port or image name not set."); @@ -272,19 +265,10 @@ int dnbd3_net_connect(dnbd3_device_t *dev) dev->panic = 0; dev->panic_count = 0; - if (get_servers) // This connection is established to the initial server (from the ioctl call) - { - // Enqueue request to request_queue_send for a fresh list of alt servers - req1->cmd_type = REQ_TYPE_SPECIAL; - req1->cmd_flags = CMD_GET_SERVERS; - list_add(&req1->queuelist, &dev->request_queue_send); - } - else if (set_client) - { - req1->cmd_type = REQ_TYPE_SPECIAL; - req1->cmd_flags = CMD_SET_CLIENT_MODE; - list_add(&req1->queuelist, &dev->request_queue_send); - } + // Enqueue request to request_queue_send for a fresh list of alt servers + req1->cmd_type = REQ_TYPE_SPECIAL; + req1->cmd_flags = CMD_GET_SERVERS; + list_add(&req1->queuelist, &dev->request_queue_send); // create required threads dev->thread_send = kthread_create(dnbd3_net_send, dev, dev->disk->disk_name); @@ -306,7 +290,8 @@ int dnbd3_net_connect(dnbd3_device_t *dev) add_timer(&dev->hb_timer); return 0; - error: if (dev->sock) + error: ; + if (dev->sock) { sock_release(dev->sock); dev->sock = NULL; @@ -340,13 +325,11 @@ int dnbd3_net_disconnect(dnbd3_device_t *dev) if (dev->thread_send) { kthread_stop(dev->thread_send); - dev->thread_send = NULL; } if (dev->thread_receive) { kthread_stop(dev->thread_receive); - dev->thread_receive = NULL; } if (dev->thread_discover) @@ -437,9 +420,10 @@ int dnbd3_net_discover(void *data) struct timeval start, end; unsigned long rtt, best_rtt = 0; unsigned long irqflags; - int i, istart, isize, best_server, current_server; + int i, j, isize, best_server, current_server; int turn = 0; - int ready = 0, do_change, last_alt_count = 0; + int ready = 0, do_change = 0; + char check_order[NUMBER_SERVERS]; int mlen; struct request *last_request = (struct request *)123, *cur_request = (struct request *)456; @@ -463,6 +447,10 @@ int dnbd3_net_discover(void *data) dnbd3_request.magic = dnbd3_packet_magic; + for (i = 0; i < NUMBER_SERVERS; ++i) { + check_order[i] = i; + } + for (;;) { wait_event_interruptible(dev->process_queue_discover, @@ -525,26 +513,34 @@ int dnbd3_net_discover(void *data) current_server = best_server = -1; best_rtt = 0xFFFFFFFul; - if (dev->heartbeat_count < STARTUP_MODE_DURATION || last_alt_count == 0 || dev->panic) + if (dev->heartbeat_count < STARTUP_MODE_DURATION || dev->panic) { - istart = 0; isize = NUMBER_SERVERS; } else { - istart = jiffies % MAX(last_alt_count - 2, 1); isize = 3; } + if (NUMBER_SERVERS > isize) { + for (i = 0; i < isize; ++i) { + j = ((start.tv_sec >> i) ^ (start.tv_usec >> j)) % NUMBER_SERVERS; + if (j != i) { + mlen = check_order[i]; + check_order[i] = check_order[j]; + check_order[j] = mlen; + } + } + } - for (i = istart; i < NUMBER_SERVERS; ++i) + for (j = 0; j < NUMBER_SERVERS; ++j) { + i = check_order[j]; if (dev->alt_servers[i].host.type == 0) // Empty slot continue; - last_alt_count = i; - if (!dev->panic && dev->alt_servers[i].failures > 50 && (jiffies & 7) != 0) // If not in panic mode, skip server if it failed too many times + if (!dev->panic && dev->alt_servers[i].failures > 50 && (start.tv_usec & 7) != 0) // If not in panic mode, skip server if it failed too many times + continue; + if (isize-- <= 0 && !is_same_server(&dev->cur_server, &dev->alt_servers[i])) continue; - if (isize-- <= 0) - break; // Initialize socket and connect if (sock_create_kern(dev->alt_servers[i].host.type, SOCK_STREAM, IPPROTO_TCP, &sock) < 0) @@ -653,13 +649,13 @@ int dnbd3_net_discover(void *data) } else if (sizeof(size_t) >= 8) { - dnbd3_request.offset = ((((start.tv_usec << 12) ^ start.tv_usec) << 4) % dev->reported_size) + dnbd3_request.offset = ((((start.tv_sec << 12) ^ start.tv_usec) << 4) % dev->reported_size) & ~(uint64_t)(RTT_BLOCK_SIZE - 1); //printk("Random offset 64bit: %lluMiB\n", (unsigned long long)(dnbd3_request.offset >> 20)); } else // On 32bit, prevent modulo on a 64bit data type. This limits the random block picking to the first 4GB of the image { - dnbd3_request.offset = ((((start.tv_usec << 12) ^ start.tv_usec) << 4) % (uint32_t)dev->reported_size) + dnbd3_request.offset = ((((start.tv_sec << 12) ^ start.tv_usec) << 4) % (uint32_t)dev->reported_size) & ~(RTT_BLOCK_SIZE - 1); //printk("Random offset 32bit: %lluMiB\n", (unsigned long long)(dnbd3_request.offset >> 20)); } @@ -727,7 +723,8 @@ int dnbd3_net_discover(void *data) continue; - error: ++dev->alt_servers[i].failures; + error: ; + ++dev->alt_servers[i].failures; sock_release(sock); sock = NULL; dev->alt_servers[i].rtts[turn] = RTT_UNREACHABLE; @@ -756,7 +753,7 @@ int dnbd3_net_discover(void *data) continue; } - do_change = !dev->is_server && ready && best_server != current_server && (jiffies & 3) != 0 + do_change = !dev->is_server && ready && best_server != current_server && (start.tv_usec & 3) != 0 && RTT_THRESHOLD_FACTOR(dev->cur_rtt) > best_rtt + 1500; if (ready && !do_change) { @@ -798,7 +795,7 @@ int dnbd3_net_discover(void *data) best_sock = NULL; } - if (!ready || (jiffies & 7) != 0) + if (!ready || (start.tv_usec & 7) != 0) turn = (turn + 1) % 4; if (turn == 3) ready = 1; @@ -811,7 +808,7 @@ int dnbd3_net_discover(void *data) int dnbd3_net_send(void *data) { dnbd3_device_t *dev = data; - struct request *blk_request; + struct request *blk_request, *tmp_request; dnbd3_request_t dnbd3_request; struct msghdr msg; @@ -825,6 +822,19 @@ int dnbd3_net_send(void *data) set_user_nice(current, -20); + // move already sent requests to request_queue_send again + while (!list_empty(&dev->request_queue_receive)) + { + printk("WARN: Request queue was not empty on %s\n", dev->disk->disk_name); + spin_lock_irqsave(&dev->blk_lock, irqflags); + list_for_each_entry_safe(blk_request, tmp_request, &dev->request_queue_receive, queuelist) + { + list_del_init(&blk_request->queuelist); + list_add(&blk_request->queuelist, &dev->request_queue_send); + } + spin_unlock_irqrestore(&dev->blk_lock, irqflags); + } + for (;;) { wait_event_interruptible(dev->process_queue_send, kthread_should_stop() || !list_empty(&dev->request_queue_send)); @@ -879,29 +889,25 @@ int dnbd3_net_send(void *data) iov.iov_len = sizeof(dnbd3_request); if (kernel_sendmsg(dev->sock, &msg, &iov, 1, sizeof(dnbd3_request)) != sizeof(dnbd3_request)) { - printk("Couldn't properly send a request header.\n"); + debug_dev("ERROR: Connection to server lost (send)"); goto error; } wake_up(&dev->process_queue_receive); } + dev->thread_send = NULL; return 0; - error: - debug_dev("ERROR: Connection to server lost (send)"); + error: ; if (dev->sock) - { kernel_sock_shutdown(dev->sock, SHUT_RDWR); - dev->sock = NULL; - } - dev->thread_send = NULL; if (!dev->disconnecting) { dev->panic = 1; - // start discover dev->discover = 1; wake_up(&dev->process_queue_discover); } + dev->thread_send = NULL; return -1; } @@ -938,7 +944,7 @@ int dnbd3_net_receive(void *data) continue; } if (ret <= 0) - error_dev_va("Connection closed (%d).", ret); + error_dev_va("ERROR: Connection to server lost (receive)", ret); if (ret != sizeof(dnbd3_reply)) error_dev("ERROR: Recv msg header."); fixup_reply(dnbd3_reply); @@ -1065,30 +1071,19 @@ int dnbd3_net_receive(void *data) } printk("dnbd3_net_receive terminated normally.\n"); + dev->thread_receive = NULL; return 0; error: - // move already sent requests to request_queue_send again - while (!list_empty(&dev->request_queue_receive)) - { - printk("WARN: Request queue was not empty on %s\n", dev->disk->disk_name); - spin_lock_irqsave(&dev->blk_lock, irqflags); - list_for_each_entry_safe(blk_request, tmp_request, &dev->request_queue_receive, queuelist) - { - list_del_init(&blk_request->queuelist); - list_add(&blk_request->queuelist, &dev->request_queue_send); - } - spin_unlock_irqrestore(&dev->blk_lock, irqflags); - } if (dev->sock) kernel_sock_shutdown(dev->sock, SHUT_RDWR); - dev->thread_receive = NULL; if (!dev->disconnecting) { dev->panic = 1; - // start discover dev->discover = 1; wake_up(&dev->process_queue_discover); } + dev->thread_receive = NULL; return -1; } + -- cgit v1.2.3-55-g7522