From 3dcda423a8c3bfeacdb11cf164a717ce7ca78572 Mon Sep 17 00:00:00 2001 From: Frederic Robra Date: Tue, 13 Aug 2019 15:13:04 +0200 Subject: changed strategy to connect to diffrent sockets --- src/kernel/net.c | 442 +++++++++++++++---------------------------------------- 1 file changed, 122 insertions(+), 320 deletions(-) (limited to 'src/kernel/net.c') diff --git a/src/kernel/net.c b/src/kernel/net.c index 5478e95..35c49b5 100644 --- a/src/kernel/net.c +++ b/src/kernel/net.c @@ -22,6 +22,7 @@ #include #include +#include #include "net.h" #include "utils.h" @@ -65,46 +66,11 @@ (h).msg_flags = MSG_WAITALL | MSG_NOSIGNAL; \ } while (0) - -#if NUMBER_CONNECTIONS == 1 -#define dnbd3_pl_socket_connect(dev, server) \ - dnbd3_socket_connect(dev, server) - -#define dnbd3_pl_socket_disconnect(sock) \ - dnbd3_socket_disconnect(sock) - - -#else -#define dnbd3_pl_socket_connect(sock, server) \ - do {\ - int i; \ - for (i = 0 < NUMBER_PARALLEL_CONNECTIONS; i++) { \ - dnbd3_socket_connect(dev, server); \ - dnbd3_socket_connect((sock) + \ - (i * sizeof(struct dnbd3_sock)), \ - (server)); \ - } \ - } while (0) - - -#define dnbd3_pl_socket_disconnect(sock) \ - do { \ - int i; \ - for (i = 0 < NUMBER_PARALLEL_CONNECTIONS; i++) { \ - dnbd3_socket_disconnect((sock) + \ - (i * sizeof(struct dnbd3_sock))); \ - } \ - } while (0) - -#endif - -static int dnbd3_server_connect(struct dnbd3_device *dev, - struct dnbd3_server *server); +static int __dnbd3_socket_connect(struct dnbd3_sock *sock, + struct dnbd3_server * server); static int dnbd3_socket_connect(struct dnbd3_sock *sock, struct dnbd3_server * server); static int dnbd3_socket_disconnect(struct dnbd3_sock *sock); -static int dnbd3_server_disconnect(struct dnbd3_device *dev, - struct dnbd3_server *server); /* @@ -216,8 +182,8 @@ int dnbd3_send_request(struct dnbd3_sock *sock, struct request *req, goto error; } - sock->pending = NULL; error: + sock->pending = NULL; return result; } @@ -738,176 +704,107 @@ static void dnbd3_keepalive_worker(struct work_struct *work) } /** - * dnbd3_find_best_alt_server - find best alternative server - * @dev: the device where to search for alternative servers - * - * searches for an alternative server which has an rtt better than RTT_THRESOLD - * of the best connected server + * dnbd3_compare_servers - comparator for the server + * @lhs: left hand sign + * @rhs: right hand sign */ -static struct dnbd3_server *dnbd3_find_best_alt_server(struct dnbd3_device *dev) -{ - int i, j; - uint64_t rtt = 0; - uint64_t best_rtt = RTT_UNREACHABLE; - uint64_t current_best_rtt = RTT_UNREACHABLE; - struct dnbd3_server *best_server = NULL; - struct dnbd3_server *server = NULL; - - for (i = 0; i < NUMBER_CONNECTIONS; i++) { - if (dnbd3_is_sock_alive(dev->socks[i])) { - rtt = dnbd3_avg_rtt(dev->socks[i].server); - if (rtt <= current_best_rtt) { - current_best_rtt = rtt; - } - } +static int dnbd3_compare_servers(const void *lhs, const void *rhs) { + uint64_t l, r; + struct dnbd3_server *lhs_server = *((struct dnbd3_server **) lhs); + struct dnbd3_server *rhs_server = *((struct dnbd3_server **) rhs); + + l = lhs_server->host.type != 0 ? lhs_server->avg_rtt + : RTT_UNREACHABLE + 1; + r = rhs_server->host.type != 0 ? rhs_server->avg_rtt + : RTT_UNREACHABLE + 1; + if (l < r) { + return -1; + } else if (l > r) { + return 1; } - - best_rtt = RTT_THRESOULD_LIMIT(current_best_rtt); - debug_dev(dev, "best connected rtt is %llu, searching for rtt better than %llu", - current_best_rtt, best_rtt); - - for (i = 0; i < NUMBER_SERVERS; i++) { - if (dev->alt_servers[i].host.type != 0) { - rtt = dnbd3_avg_rtt(&dev->alt_servers[i]); - if (rtt <= best_rtt) { - server = &dev->alt_servers[i]; - /* check if already connected */ - for (j = 0; j < NUMBER_CONNECTIONS; j++) { - if (server == dev->socks[j].server) { - server = NULL; - break; - } - } - if (server) { - best_server = server; - best_rtt = rtt; - } - } - } - } - if (best_server) { - debug_server(dev, best_server, "found best alt server with rtt %llu", - best_rtt); - } else { - debug_dev(dev, "did not find any alternative server"); - } - return best_server; + return 0; } /** - * dnbd3_better_rtt - checks if the rtt is better - * @new_server: the server to check - * @existing_server: current server + * dnbd3_sort_server - sort the alt server according to their avg rtt + * @dev: the dndb3 device * - * checks if the rtt is better than RTT_THRESHOLD_FACTOR + * the returned array has to be freed with kfree */ -static bool dnbd3_better_rtt(struct dnbd3_server *new_server, - struct dnbd3_server *existing_server) { - uint64_t new_rtt = dnbd3_avg_rtt(new_server); - uint64_t existing_rtt = dnbd3_avg_rtt(existing_server); +static struct dnbd3_server **dnbd3_sort_server(struct dnbd3_device *dev) { + int i; + struct dnbd3_server **sorted_servers = kmalloc(NUMBER_SERVERS * sizeof(struct dnbd3_device *), GFP_KERNEL); + if (!sorted_servers) { + return NULL; + } - if (new_rtt < RTT_THRESHOLD_FACTOR(existing_rtt)) { - return true; + for (i = 0; i < NUMBER_SERVERS; i++) { + sorted_servers[i] = &dev->alt_servers[i]; } - return false; + sort(sorted_servers, NUMBER_SERVERS, sizeof(struct dnbd3_device *), + &dnbd3_compare_servers, NULL); + + return sorted_servers; } /** - * dnbd3_adjust_connections - adjust the connections of the device - * @dev: the device + * dnbd3_adjust_connections - create a connection plan and connect + * @dev: the dnbd3 device * - * 1. connect empty sockets if best alternative server is found - * 2. replace slow socket with better server if available - * 3. remove socket if one is slow + * 1. sort the alt server after the avg rtt + * 2. create a connection plan + * 3. connect the plan */ -static void dnbd3_adjust_connections(struct dnbd3_device *dev) { - int i; - int sock_alive = 0; - uint64_t rtt; - uint64_t best_rtt = RTT_UNREACHABLE; - struct dnbd3_server *server, *existing_server; +static int dnbd3_adjust_connections(struct dnbd3_device *dev) { + int i, j, fallback; + struct dnbd3_server *plan[NUMBER_CONNECTIONS]; + struct dnbd3_server **servers = dnbd3_sort_server(dev); - /* connect empty sockets */ //TODO check if panic worker connects at the same time, than we would have a problem - sock_alive = 0; - for (i = 0; i < NUMBER_CONNECTIONS; i++) { - if (!dnbd3_is_sock_alive(dev->socks[i])) { - server = dnbd3_find_best_alt_server(dev); - if (server) { - if (dnbd3_server_connect(dev, server) == 0) { - sock_alive++; + if (servers && servers[0]->host.type != 0) { + mutex_lock(&dev->device_lock); + plan[0] = servers[0]; + fallback = 0; + j = 1; + debug_dev(dev, "connection plan:"); + debug_server(dev, plan[0], "server 0 with rtt %llu:", + plan[0]->avg_rtt); + for (i = 1; i < NUMBER_CONNECTIONS; i++) { + if (servers[j]->host.type != 0) { + if (RTT_FACTOR(plan[i - 1]->avg_rtt) > + servers[j]->avg_rtt) { + plan[i] = servers[j]; + j++; } else { - warn_server(dev, server, "failed to connect"); + plan[i] = plan[fallback]; + fallback++; } + } else { + plan[i] = plan[fallback]; + fallback++; } - } else { - sock_alive++; + debug_server(dev, plan[i], "server %d with rtt %llu:", + i, plan[i]->avg_rtt); } - } + kfree(servers); - /* replace socket with better server */ - if (sock_alive == NUMBER_CONNECTIONS) { for (i = 0; i < NUMBER_CONNECTIONS; i++) { - if (dnbd3_is_sock_alive(dev->socks[i])) { - server = dnbd3_find_best_alt_server(dev); - existing_server = dev->socks[i].server; - if (server && dnbd3_better_rtt(server, dev->socks[i].server)) { + if (plan[i] != dev->socks[i].server) { + if (dnbd3_is_sock_alive(dev->socks[i])) { dnbd3_socket_disconnect(&dev->socks[i]); - - /* if it fails reconnect to existing */ - if (dnbd3_server_connect(dev, server) != 0) { - warn_server(dev, server, "failed to connect"); - dnbd3_server_connect(dev, existing_server); - } - } - } - } - } - - /* remove a socket if it is much slower than the others */ - if (sock_alive > 1) { - for (i = 0; i < NUMBER_CONNECTIONS; i++) { - if (dnbd3_is_sock_alive(dev->socks[i])) { - rtt = dnbd3_avg_rtt(dev->socks[i].server); - if (rtt <= best_rtt) { - best_rtt = rtt; } + dnbd3_socket_connect(&dev->socks[i], plan[i]); } } - for (i = 0; i < NUMBER_CONNECTIONS; i++) { - if (dnbd3_is_sock_alive(dev->socks[i])) { - rtt = dnbd3_avg_rtt(dev->socks[i].server); - if (rtt > RTT_THRESOULD_LIMIT(best_rtt)) { - info_sock(&dev->socks[i], "removing connection with rtt %llu", rtt); - dnbd3_socket_disconnect(&dev->socks[i]); - sock_alive--; - } - } + mutex_unlock(&dev->device_lock); + return 0; + } else { /* there is nothing to connect */ + if (servers) { + kfree(servers); } + return -ENONET; } - info_dev(dev, "connected to %d/%d sockets", sock_alive, NUMBER_CONNECTIONS); -} -/** - * dnbd3_panic_connect - connect to the first available server - * @dev: the device - */ -static int dnbd3_panic_connect(struct dnbd3_device *dev) { - int result, i; - result = dnbd3_server_connect(dev, &dev->initial_server); - if (result) { - for (i = 0; i < NUMBER_SERVERS; i++) { - if (dev->alt_servers[i].host.type != 0) { - result = dnbd3_server_connect(dev, - &dev->alt_servers[i]); - if (!result) { - info_server(dev, &dev->alt_servers[i], "found server to connect to"); - break; - } - } - } - } - return result; } /** @@ -921,47 +818,27 @@ static int dnbd3_panic_connect(struct dnbd3_device *dev) { static void dnbd3_panic_worker(struct work_struct *work) { struct dnbd3_device *dev; - struct dnbd3_sock *panicked_sock = NULL; - struct dnbd3_server *new_server, *panicked_server; + bool panic = false; int i; int sock_alive = 0; dev = container_of(work, struct dnbd3_device, panic_worker); for (i = 0; i < NUMBER_CONNECTIONS; i++) { if (dev->socks[i].panic) { - panicked_sock = &dev->socks[i]; + panic = true; + dnbd3_set_rtt_unreachable(dev->socks[i].server); + dnbd3_socket_disconnect(&dev->socks[i]); + } else if (dnbd3_is_sock_alive(dev->socks[i])) { sock_alive++; } } - if (panicked_sock) { - warn_sock(panicked_sock, "panicked, connections still alive %d", - sock_alive); - panicked_server = panicked_sock->server; - new_server = dnbd3_find_best_alt_server(dev); - dnbd3_socket_disconnect(panicked_sock); - dnbd3_set_rtt_unreachable(panicked_server); + if (panic) { + warn_dev(dev, "panicked, connections still alive %d", + sock_alive); - if (new_server != NULL && new_server != panicked_server) { - info_server(dev, new_server, "found replacement"); - if (!dnbd3_server_connect(dev, new_server)) { - sock_alive++; - } - } else if (sock_alive > 0) { - info_dev(dev, "found no replacement server but still connected to %d servers", - sock_alive); - } - } - - if (sock_alive == 0) { - warn_dev(dev, "did not find a good server, trying to connect to any available server"); - - if (dnbd3_panic_connect(dev)) { - error_dev(dev, "could not connect to any server"); - } else { - info_dev(dev, "found server to connect to"); - } + dnbd3_adjust_connections(dev); } } @@ -985,7 +862,7 @@ static int dnbd3_meassure_rtt(struct dnbd3_device *dev, .server = server }; - result = dnbd3_socket_connect(&sock, server); + result = __dnbd3_socket_connect(&sock, server); if (result) { error_sock(&sock, "socket connect failed in rtt measurement"); goto error; @@ -1044,6 +921,7 @@ static int dnbd3_meassure_rtt(struct dnbd3_device *dev, error: sock.server->rtts[dev->discovery_count % 4] = rtt; + sock.server->avg_rtt = dnbd3_avg_rtt(sock.server); if (result <= 0) { server->failures++; } @@ -1092,8 +970,9 @@ static void dnbd3_merge_new_server(struct dnbd3_device *dev, if (new_server->failures == 1) { /* remove is requested */ info_server(dev, new_server, "remove server is requested"); - dnbd3_server_disconnect(dev, existing_server); + // adjust connection will remove it later existing_server->host.type = 0; + dnbd3_set_rtt_unreachable(existing_server); } // existing_server->failures = 0; // reset failure count return; @@ -1110,9 +989,10 @@ static void dnbd3_merge_new_server(struct dnbd3_device *dev, info_server(dev, free_server, "got new alternative server"); free_server->failures = 0; free_server->protocol_version = 0; - dnbd3_set_rtt_unreachable(free_server); + dnbd3_set_rtt_unknown(free_server); } + /** * dnbd3_discovery_worker - handle discovery * @work: the work used to get the dndb3_device @@ -1168,11 +1048,11 @@ static void dnbd3_discovery_worker(struct work_struct *work) /** - * dnbd3_socket_connect - connect a socket to a server + * __dnbd3_socket_connect - internal connect a socket to a server * @sock: the socket to connect * @server: the server */ -static int dnbd3_socket_connect(struct dnbd3_sock *sock, +static int __dnbd3_socket_connect(struct dnbd3_sock *sock, struct dnbd3_server *server) { int result = 0; @@ -1237,44 +1117,27 @@ error: } /** - * dnbd3_server_connect - connect a server to a device - * @dev: the device + * dnbd3_socket_connect - connect a socket to a server + * @sock: the socket * @server: the server to connect * - * 1. connects the server to a free socket if available + * 1. connects the server to the socket * 2. select the image * 3. start receiver_worker and keepalive_worker - * 4. if it is the first connection start timer, panic_worker and - * keepalive_worker - * 5. update the mq queues to the number of sockets alive */ -static int dnbd3_server_connect(struct dnbd3_device *dev, +static int dnbd3_socket_connect(struct dnbd3_sock *sock, struct dnbd3_server *server) { - int i; - int sock_alive = 0; int result = -EIO; dnbd3_reply_t reply; - struct dnbd3_sock *sock = NULL; - for (i = 0; i < NUMBER_CONNECTIONS; i++) { - if (!dnbd3_is_sock_alive(dev->socks[i])) { - sock = &dev->socks[i]; - break; - } - } - if (sock == NULL) { - warn_server(dev, server, "could not connect to socket, to many connections"); - return -EIO; - } + struct dnbd3_device *dev = sock->device; sock->server = server; - debug_sock(sock, "socket connect"); - mutex_init(&sock->tx_lock); mutex_lock(&sock->tx_lock); - result = dnbd3_socket_connect(sock, server); + result = __dnbd3_socket_connect(sock, server); if (result) { error_sock(sock, "connection to socket failed"); result = -EIO; @@ -1294,6 +1157,7 @@ static int dnbd3_server_connect(struct dnbd3_device *dev, result = dnbd3_send_request_cmd(sock, CMD_SELECT_IMAGE); if (result <= 0) { + error_sock(sock, "connection to image %s failed", dev->imgname); result = -EIO; goto error; @@ -1328,23 +1192,7 @@ static int dnbd3_server_connect(struct dnbd3_device *dev, INIT_WORK(&sock->keepalive_worker, dnbd3_keepalive_worker); - for (i = 0; i < NUMBER_CONNECTIONS; i++) { - if (dev->socks[i].sock && dev->socks[i].server) { - sock_alive++; - } - } - /* if first socket to connect, start timer and workers */ - if (sock_alive == 1) { - debug_sock(sock, "first connection to server, starting workers"); - INIT_WORK(&dev->discovery_worker, dnbd3_discovery_worker); - INIT_WORK(&dev->panic_worker, dnbd3_panic_worker); - timer_setup(&dev->timer, dnbd3_timer, 0); - dev->timer.expires = jiffies + HZ; - add_timer(&dev->timer); - } - blk_mq_update_nr_hw_queues(&dev->tag_set, sock_alive); - - /* request alternative servers receiver will handle this */ + /* TODO not on every connect? request alternative servers receiver will handle this */ if (dnbd3_send_request_cmd(sock, CMD_GET_SERVERS) <= 0) { error_sock(sock, "failed to get servers in discovery"); } @@ -1352,6 +1200,7 @@ static int dnbd3_server_connect(struct dnbd3_device *dev, return 0; error: server->failures++; + sock->panic = true; if (sock->sock) { kernel_sock_shutdown(sock->sock, SHUT_RDWR); cancel_work_sync(&sock->receive_worker); @@ -1362,41 +1211,13 @@ error: return result; } -static void dnbd3_clear_req(struct request *req, void *data, bool reserved) -{ - struct dnbd3_cmd *cmd = blk_mq_rq_to_pdu(req); - - cmd->status = BLK_STS_IOERR; - blk_mq_complete_request(req); -} /** - * dnbd3_socket_disconnect - disconnect a socket or server + * dnbd3_socket_disconnect - disconnect a socket * @sock: the socket to disconnect - * - * 1. update nr of mq queues - * 2. if last socket remove timer - * 3. disconnect socket */ static int dnbd3_socket_disconnect(struct dnbd3_sock *sock) { - int i; - struct dnbd3_device *dev = sock->device; - int sock_alive = 0; - for (i = 0; i < NUMBER_CONNECTIONS; i++) { - if (dnbd3_is_sock_alive(dev->socks[i])) { - sock_alive++; - } - } - if (sock_alive <= 1) { - info_sock(sock, "shutting down last socket and stopping timer"); - del_timer_sync(&dev->timer); - /* - * do not wait for discovery and panic worker as they may have - * called this method - */ - - } cancel_work_sync(&sock->keepalive_worker); debug_sock(sock, "socket disconnect"); @@ -1425,50 +1246,18 @@ static int dnbd3_socket_disconnect(struct dnbd3_sock *sock) sock_release(sock->sock); sock->sock = NULL; } - debug_sock(sock, "update nr hw queues to %d", sock_alive -1); -// blk_mq_quiesce_queue(dev->disk->queue); -// debug_sock(sock, "busy iter"); -// blk_mq_tagset_busy_iter(&dev->tag_set, dnbd3_clear_req, NULL); -// debug_sock(sock, "update"); - //TODO can we just stop the failed queue? - blk_mq_update_nr_hw_queues(&dev->tag_set, sock_alive - 1); -// debug_sock(sock, "unquisco"); -// blk_mq_unquiesce_queue(dev->disk->queue); -// debug_sock(sock, "donee"); sock->server = NULL; sock->panic = false; return 0; } -/** - * dnbd3_server_disconnect - disconnect a server from a socket - * @dev: the device - * @server: the server to disconnect - */ -static int dnbd3_server_disconnect(struct dnbd3_device *dev, - struct dnbd3_server *server) -{ - int i; - struct dnbd3_sock *sock = NULL; - for (i = 0; i < NUMBER_CONNECTIONS; i++) { - if (dev->socks[i].server == server) { - sock = &dev->socks[i]; - } - } - if (!sock || !sock->sock) { - warn_dev(dev, "could not find socket to disconnect"); - return -EIO; - } - return dnbd3_socket_disconnect(sock); -} - /** * dnbd3_net_connect - connect device * @dev: the device to connect * - * dnbd3_device.alt_servers[0] must set + * dnbd3_device.alt_servers[0] must be set */ int dnbd3_net_connect(struct dnbd3_device *dev) { @@ -1479,14 +1268,27 @@ int dnbd3_net_connect(struct dnbd3_device *dev) return -ENONET; } - // alt_server[0] is the initial server - result = dnbd3_server_connect(dev, &dev->alt_servers[0]); + result = dnbd3_adjust_connections(dev); if (result) { error_dev(dev, "failed to connect to initial server"); - result = -ENOENT; - dev->imgname = NULL; - dev->socks[0].server = NULL; + return -ENOENT; } + + debug_dev(dev, "connected, starting workers"); + INIT_WORK(&dev->discovery_worker, dnbd3_discovery_worker); + INIT_WORK(&dev->panic_worker, dnbd3_panic_worker); + timer_setup(&dev->timer, dnbd3_timer, 0); + dev->timer.expires = jiffies + HZ; + add_timer(&dev->timer); + + // alt_server[0] is the initial server +// result = dnbd3_server_connect(dev, &dev->alt_servers[0]); +// if (result) { +// error_dev(dev, "failed to connect to initial server"); +// result = -ENOENT; +// dev->imgname = NULL; +// dev->socks[0].server = NULL; +// } return result; } -- cgit v1.2.3-55-g7522