From ef92307fd49e75482c7599caf68685afc1807512 Mon Sep 17 00:00:00 2001 From: Manuel Bentele Date: Fri, 6 Nov 2020 12:44:43 +0100 Subject: [KERNEL, CLIENT]: submit and probe multiple dnbd3-server with ioctl OPEN The ioctl OPEN call for DNBD3 devices exposed by the dnbd3 Linux kernel module, is extended with a fixed array of dnbd3 hosts. The fixed array allows the dnbd3-client to submit host information (IP address and port) of multiple dnbd3-servers. This information is used to probe all submitted dnbd3-servers and add them to the alternative dnbd3-server list. If at least one dnbd3-server is not reachable, the OPEN ioctl call will abort with an error code. --- src/kernel/net.c | 244 ++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 168 insertions(+), 76 deletions(-) (limited to 'src/kernel/net.c') diff --git a/src/kernel/net.c b/src/kernel/net.c index 46c369a..57d8cc7 100644 --- a/src/kernel/net.c +++ b/src/kernel/net.c @@ -27,7 +27,6 @@ #include #include -#include #ifndef MIN #define MIN(a,b) ((a) < (b) ? (a) : (b)) @@ -230,7 +229,7 @@ static int dnbd3_net_discover(void *data) check_order[i] = i; } - for (;;) + while (!kthread_should_stop()) { wait_event_interruptible(dev->process_queue_discover, kthread_should_stop() || dev->discover || dev->thread_discover == NULL); @@ -246,9 +245,9 @@ static int dnbd3_net_discover(void *data) continue; // Check if the list of alt servers needs to be updated and do so if necessary + spin_lock_irqsave(&dev->blk_lock, irqflags); if (dev->new_servers_num) { - spin_lock_irqsave(&dev->blk_lock, irqflags); for (i = 0; i < dev->new_servers_num; ++i) { if (dev->new_servers[i].host.type != HOST_IP4 && dev->new_servers[i].host.type != HOST_IP6) // Invalid entry? @@ -286,8 +285,8 @@ static int dnbd3_net_discover(void *data) alt_server->failures = 0; } dev->new_servers_num = 0; - spin_unlock_irqrestore(&dev->blk_lock, irqflags); } + spin_unlock_irqrestore(&dev->blk_lock, irqflags); current_server = best_server = -1; best_rtt = 0xFFFFFFFul; @@ -611,8 +610,9 @@ static int dnbd3_net_discover(void *data) } - dev_dbg(dnbd3_device_to_dev(dev), "kthread dnbd3_net_discover terminated normally\n"); kfree(buf); + dev_dbg(dnbd3_device_to_dev(dev), "kthread dnbd3_net_discover terminated normally\n"); + dev->thread_discover = NULL; return 0; } @@ -634,19 +634,19 @@ static int dnbd3_net_send(void *data) set_user_nice(current, -20); // move already sent requests to request_queue_send again - while (!list_empty(&dev->request_queue_receive)) + spin_lock_irqsave(&dev->blk_lock, irqflags); + if (!list_empty(&dev->request_queue_receive)) { dev_warn(dnbd3_device_to_dev(dev), "request queue was not empty"); - spin_lock_irqsave(&dev->blk_lock, irqflags); list_for_each_entry_safe(blk_request, tmp_request, &dev->request_queue_receive, queuelist) { list_del_init(&blk_request->queuelist); list_add(&blk_request->queuelist, &dev->request_queue_send); } - spin_unlock_irqrestore(&dev->blk_lock, irqflags); } + spin_unlock_irqrestore(&dev->blk_lock, irqflags); - for (;;) + while (!kthread_should_stop()) { wait_event_interruptible(dev->process_queue_send, kthread_should_stop() || !list_empty(&dev->request_queue_send)); @@ -654,6 +654,7 @@ static int dnbd3_net_send(void *data) break; // extract block request + /* lock since we aquire a blk request from the request_queue_send */ spin_lock_irqsave(&dev->blk_lock, irqflags); if (list_empty(&dev->request_queue_send)) { @@ -661,7 +662,6 @@ static int dnbd3_net_send(void *data) continue; } blk_request = list_entry(dev->request_queue_send.next, struct request, queuelist); - spin_unlock_irqrestore(&dev->blk_lock, irqflags); // what to do? switch (dnbd3_req_op(blk_request)) @@ -671,22 +671,17 @@ static int dnbd3_net_send(void *data) dnbd3_request.offset = blk_rq_pos(blk_request) << 9; // *512 dnbd3_request.size = blk_rq_bytes(blk_request); // bytes left to complete entire request // enqueue request to request_queue_receive - spin_lock_irqsave(&dev->blk_lock, irqflags); list_del_init(&blk_request->queuelist); list_add_tail(&blk_request->queuelist, &dev->request_queue_receive); - spin_unlock_irqrestore(&dev->blk_lock, irqflags); break; case DNBD3_REQ_OP_SPECIAL: dnbd3_request.cmd = dnbd3_priv_to_cmd(blk_request); dnbd3_request.size = 0; - spin_lock_irqsave(&dev->blk_lock, irqflags); list_del_init(&blk_request->queuelist); - spin_unlock_irqrestore(&dev->blk_lock, irqflags); break; default: dev_err(dnbd3_device_to_dev(dev), "unknown command (send %u %u)\n", (int)blk_request->cmd_flags, (int)dnbd3_req_op(blk_request)); - spin_lock_irqsave(&dev->blk_lock, irqflags); list_del_init(&blk_request->queuelist); spin_unlock_irqrestore(&dev->blk_lock, irqflags); continue; @@ -694,6 +689,7 @@ static int dnbd3_net_send(void *data) // send net request dnbd3_request.handle = (uint64_t)(uintptr_t)blk_request; // Double cast to prevent warning on 32bit + spin_unlock_irqrestore(&dev->blk_lock, irqflags); fixup_request(dnbd3_request); iov.iov_base = &dnbd3_request; iov.iov_len = sizeof(dnbd3_request); @@ -702,14 +698,13 @@ static int dnbd3_net_send(void *data) dnbd3_dev_err_host_cur(dev, "connection to server lost (send)\n"); goto error; } - wake_up(&dev->process_queue_receive); } dev_dbg(dnbd3_device_to_dev(dev), "kthread dnbd3_net_send terminated normally\n"); dev->thread_send = NULL; return 0; - error: ; +error: if (dev->sock) kernel_sock_shutdown(dev->sock, SHUT_RDWR); if (!dev->disconnecting) @@ -718,6 +713,7 @@ static int dnbd3_net_send(void *data) dev->discover = 1; wake_up(&dev->process_queue_discover); } + dev_err(dnbd3_device_to_dev(dev), "kthread dnbd3_net_send terminated abnormally\n"); dev->thread_send = NULL; return -1; } @@ -735,11 +731,10 @@ static int dnbd3_net_receive(void *data) struct bio_vec *bvec = &bvec_inst; void *kaddr; unsigned long irqflags; - sigset_t blocked, oldset; uint16_t rid; unsigned long int recv_timeout = jiffies; - int count, remaining, ret; + int count, remaining, ret = 0; init_msghdr(msg); set_user_nice(current, -20); @@ -750,25 +745,44 @@ static int dnbd3_net_receive(void *data) iov.iov_base = &dnbd3_reply; iov.iov_len = sizeof(dnbd3_reply); ret = kernel_recvmsg(dev->sock, &msg, &iov, 1, sizeof(dnbd3_reply), msg.msg_flags); - if (ret == -EAGAIN) + + /* end thread after socket timeout or reception of data */ + if (kthread_should_stop()) + break; + + /* check return value of kernel_recvmsg() */ + if (ret == 0) { - if (jiffies < recv_timeout) recv_timeout = jiffies; // Handle overflow - if ((jiffies - recv_timeout) / HZ > SOCKET_KEEPALIVE_TIMEOUT) - { - dnbd3_dev_err_host_cur(dev, "receive timeout reached (%d of %d secs)\n", (int)((jiffies - recv_timeout) / HZ), (int)SOCKET_KEEPALIVE_TIMEOUT); - goto error; - } - continue; + /* have not received any data, but remote peer is shutdown properly */ + dnbd3_dev_dbg_host_cur(dev, "remote peer has performed an orderly shutdown\n"); + goto cleanup; } - if (ret <= 0) + else if (ret < 0) { - dnbd3_dev_err_host_cur(dev, "connection to server lost (receive)\n"); - goto error; + if (ret == -EAGAIN) + { + if (jiffies < recv_timeout) recv_timeout = jiffies; // Handle overflow + if ((jiffies - recv_timeout) / HZ > SOCKET_KEEPALIVE_TIMEOUT) + { + dnbd3_dev_err_host_cur(dev, "receive timeout reached (%d of %d secs)\n", (int)((jiffies - recv_timeout) / HZ), (int)SOCKET_KEEPALIVE_TIMEOUT); + ret = -ETIMEDOUT; + goto cleanup; + } + continue; + } else { + /* for all errors other than -EAGAIN, print message and abort thread */ + dnbd3_dev_err_host_cur(dev, "connection to server lost (receive)\n"); + ret = -ESHUTDOWN; + goto cleanup; + } } + + /* check if arrived data is valid */ if (ret != sizeof(dnbd3_reply)) { dnbd3_dev_err_host_cur(dev, "recv msg header\n"); - goto error; + ret = -EINVAL; + goto cleanup; } fixup_reply(dnbd3_reply); @@ -776,12 +790,14 @@ static int dnbd3_net_receive(void *data) if (dnbd3_reply.magic != dnbd3_packet_magic) { dnbd3_dev_err_host_cur(dev, "wrong packet magic (receive)\n"); - goto error; + ret = -EINVAL; + goto cleanup; } if (dnbd3_reply.cmd == 0) { dnbd3_dev_err_host_cur(dev, "command was 0 (Receive)\n"); - goto error; + ret = -EINVAL; + goto cleanup; } // Update timeout @@ -807,27 +823,23 @@ static int dnbd3_net_receive(void *data) { dnbd3_dev_err_host_cur(dev, "received block data for unrequested handle (%llu: %llu)\n", (unsigned long long)dnbd3_reply.handle, (unsigned long long)dnbd3_reply.size); - goto error; + ret = -EINVAL; + goto cleanup; } // receive data and answer to block layer rq_for_each_segment(bvec_inst, blk_request, iter) { - siginitsetinv(&blocked, sigmask(SIGKILL)); - sigprocmask(SIG_SETMASK, &blocked, &oldset); - kaddr = kmap(bvec->bv_page) + bvec->bv_offset; iov.iov_base = kaddr; iov.iov_len = bvec->bv_len; if (kernel_recvmsg(dev->sock, &msg, &iov, 1, bvec->bv_len, msg.msg_flags) != bvec->bv_len) { kunmap(bvec->bv_page); - sigprocmask(SIG_SETMASK, &oldset, NULL ); dnbd3_dev_err_host_cur(dev, "receiving from net to block layer\n"); - goto error; + ret = -EINVAL; + goto cleanup; } kunmap(bvec->bv_page); - - sigprocmask(SIG_SETMASK, &oldset, NULL ); } spin_lock_irqsave(&dev->blk_lock, irqflags); list_del_init(&blk_request->queuelist); @@ -854,7 +866,8 @@ static int dnbd3_net_receive(void *data) != (count * sizeof(dnbd3_server_entry_t))) { dnbd3_dev_err_host_cur(dev, "recv CMD_GET_SERVERS payload\n"); - goto error; + ret = -EINVAL; + goto cleanup; } spin_lock_irqsave(&dev->blk_lock, irqflags); dev->new_servers_num = count; @@ -871,7 +884,8 @@ static int dnbd3_net_receive(void *data) if (ret <= 0) { dnbd3_dev_err_host_cur(dev, "recv additional payload from CMD_GET_SERVERS\n"); - goto error; + ret = -EINVAL; + goto cleanup; } remaining -= ret; } @@ -909,11 +923,9 @@ static int dnbd3_net_receive(void *data) } } - dev_dbg(dnbd3_device_to_dev(dev), "kthread dnbd3_net_receive terminated normally\n"); - dev->thread_receive = NULL; - return 0; + goto out; -error: +cleanup: if (dev->sock) kernel_sock_shutdown(dev->sock, SHUT_RDWR); if (!dev->disconnecting) @@ -922,8 +934,14 @@ error: dev->discover = 1; wake_up(&dev->process_queue_discover); } + +out: + if (!ret) + dev_dbg(dnbd3_device_to_dev(dev), "kthread dnbd3_net_receive terminated normally\n"); + else + dev_err(dnbd3_device_to_dev(dev), "kthread dnbd3_net_receive terminated abnormally\n"); dev->thread_receive = NULL; - return -1; + return ret; } int dnbd3_net_connect(dnbd3_device_t *dev) @@ -940,20 +958,13 @@ int dnbd3_net_connect(dnbd3_device_t *dev) char *timeout_ptr; #endif - if (dev->disconnecting) { - dnbd3_dev_dbg_host_cur(dev, "CONNECT: still disconnecting!\n"); + if (dev->disconnecting) + { + dnbd3_dev_dbg_host_cur(dev, "connect: wait for disconnect has finished ...\n"); + set_current_state(TASK_INTERRUPTIBLE); while (dev->disconnecting) schedule(); - } - if (dev->thread_receive != NULL) { - dnbd3_dev_dbg_host_cur(dev, "CONNECT: still receiving!\n"); - while (dev->thread_receive != NULL) - schedule(); - } - if (dev->thread_send != NULL) { - dnbd3_dev_dbg_host_cur(dev, "CONNECT: still sending!\n"); - while (dev->thread_send != NULL) - schedule(); + dnbd3_dev_dbg_host_cur(dev, "connect: disconnect is done\n"); } timeout.tv_sec = SOCKET_TIMEOUT_CLIENT_DATA; @@ -1151,13 +1162,40 @@ int dnbd3_net_connect(dnbd3_device_t *dev) list_add(&req1->queuelist, &dev->request_queue_send); // create required threads - dev->thread_send = kthread_create(dnbd3_net_send, dev, dev->disk->disk_name); - dev->thread_receive = kthread_create(dnbd3_net_receive, dev, dev->disk->disk_name); - dev->thread_discover = kthread_create(dnbd3_net_discover, dev, dev->disk->disk_name); + dev->thread_send = kthread_create(dnbd3_net_send, dev, "%s-send", dev->disk->disk_name); + dev->thread_receive = kthread_create(dnbd3_net_receive, dev, "%s-receive", dev->disk->disk_name); + dev->thread_discover = kthread_create(dnbd3_net_discover, dev, "%s-discover", dev->disk->disk_name); + // start them up - wake_up_process(dev->thread_send); - wake_up_process(dev->thread_receive); - wake_up_process(dev->thread_discover); + if (!IS_ERR(dev->thread_send)) { + get_task_struct(dev->thread_send); + wake_up_process(dev->thread_send); + } else { + dev_err(dnbd3_device_to_dev(dev), "failed to create send thread\n"); + /* reset error to cleanup thread */ + dev->thread_send = NULL; + goto cleanup_thread; + } + + if (!IS_ERR(dev->thread_receive)) { + get_task_struct(dev->thread_receive); + wake_up_process(dev->thread_receive); + } else { + dev_err(dnbd3_device_to_dev(dev), "failed to create receive thread\n"); + /* reset error to cleanup thread */ + dev->thread_receive = NULL; + goto cleanup_thread; + } + + if (!IS_ERR(dev->thread_discover)) { + get_task_struct(dev->thread_discover); + wake_up_process(dev->thread_discover); + } else { + dev_err(dnbd3_device_to_dev(dev), "failed to create discover thread\n"); + /* reset error to cleanup thread */ + dev->thread_discover = NULL; + goto cleanup_thread; + } wake_up(&dev->process_queue_send); @@ -1169,6 +1207,9 @@ int dnbd3_net_connect(dnbd3_device_t *dev) return 0; +cleanup_thread: + dnbd3_net_disconnect(dev); + error: if (dev->sock) { @@ -1185,14 +1226,19 @@ error: int dnbd3_net_disconnect(dnbd3_device_t *dev) { - if (dev->disconnecting) - return 0; + struct task_struct* thread = NULL; + bool thread_not_terminated = false; + int ret = 0; - if (dev->cur_server.host.port) - dnbd3_dev_dbg_host_cur(dev, "disconnecting device\n"); + if (dev->disconnecting) { + ret = -EBUSY; + goto out; + } dev->disconnecting = 1; + dev_dbg(dnbd3_device_to_dev(dev), "disconnecting device ...\n"); + // clear heartbeat timer del_timer(&dev->hb_timer); @@ -1204,21 +1250,58 @@ int dnbd3_net_disconnect(dnbd3_device_t *dev) // kill sending and receiving threads if (dev->thread_send) { - kthread_stop(dev->thread_send); + dnbd3_dev_dbg_host_cur(dev, "stop send thread\n"); + thread = dev->thread_send; + ret = kthread_stop(thread); + put_task_struct(thread); + if (ret == -EINTR) { + /* thread has never been scheduled and run */ + dev_dbg(dnbd3_device_to_dev(dev), "send thread has never run\n"); + } else { + /* thread has run, check if it has terminated successfully */ + if (dev->thread_send != NULL) { + dev_err(dnbd3_device_to_dev(dev), "send thread was not terminated correctly\n"); + thread_not_terminated = true; + } + } } if (dev->thread_receive) { - kthread_stop(dev->thread_receive); + dnbd3_dev_dbg_host_cur(dev, "stop receive thread\n"); + thread = dev->thread_receive; + ret = kthread_stop(thread); + put_task_struct(thread); + if (ret == -EINTR) { + /* thread has never been scheduled and run */ + dev_dbg(dnbd3_device_to_dev(dev), "receive thread has never run\n"); + } else { + /* thread has run, check if it has terminated successfully */ + if (dev->thread_receive != NULL) { + dev_err(dnbd3_device_to_dev(dev), "receive thread was not terminated correctly\n"); + thread_not_terminated = true; + } + } } if (dev->thread_discover) { - kthread_stop(dev->thread_discover); - dev->thread_discover = NULL; + dnbd3_dev_dbg_host_cur(dev, "stop discover thread\n"); + thread = dev->thread_discover; + ret = kthread_stop(thread); + put_task_struct(thread); + if (ret == -EINTR) { + /* thread has never been scheduled and run */ + dev_dbg(dnbd3_device_to_dev(dev), "discover thread has never run\n"); + } else { + /* thread has run, check if it has terminated successfully */ + if (dev->thread_discover != NULL) { + dev_err(dnbd3_device_to_dev(dev), "discover thread was not terminated correctly\n"); + thread_not_terminated = true; + } + } } - // clear socket if (dev->sock) { sock_release(dev->sock); @@ -1227,7 +1310,16 @@ int dnbd3_net_disconnect(dnbd3_device_t *dev) dev->cur_server.host.type = 0; dev->cur_server.host.port = 0; + if (thread_not_terminated) { + dev_err(dnbd3_device_to_dev(dev), "failed to disconnect device\n"); + ret = -ENODEV; + } else { + dev_dbg(dnbd3_device_to_dev(dev), "device is disconnected\n"); + ret = 0; + } + dev->disconnecting = 0; - return 0; +out: + return ret; } -- cgit v1.2.3-55-g7522