diff options
| -rw-r--r-- | src/kernel/core.c | 61 | ||||
| -rw-r--r-- | src/kernel/dnbd3.h | 13 | ||||
| -rw-r--r-- | src/kernel/net.c | 442 | ||||
| -rw-r--r-- | src/kernel/sysfs.c | 40 |
4 files changed, 175 insertions, 381 deletions
diff --git a/src/kernel/core.c b/src/kernel/core.c index 8bb23d9..6f0dab4 100644 --- a/src/kernel/core.c +++ b/src/kernel/core.c @@ -72,6 +72,8 @@ static void dnbd3_requeue_cmd(struct dnbd3_cmd *cmd) if (!cmd->requed) { cmd->requed = true; blk_mq_requeue_request(req, true); + } else { + blk_mq_end_request(req, BLK_STS_IOERR); } } @@ -84,47 +86,22 @@ static int dnbd3_handle_cmd(struct dnbd3_cmd *cmd, int index) { struct request *req = blk_mq_rq_from_pdu(cmd); struct dnbd3_device *dev = cmd->dnbd3; - struct dnbd3_sock *sock = NULL; - bool first_try = true; + struct dnbd3_sock *sock = &dev->socks[index]; int ret = -1; - int i; - int sock_alive = 0; - debug_dev(dev, "handle request at position %lu, size %d, index %d", - blk_rq_pos(req), blk_rq_bytes(req), index); +// debug_dev(dev, "handle request at position %lu, size %d, index %d", +// blk_rq_pos(req), blk_rq_bytes(req), index); -again: - for (i = 0; i < NUMBER_CONNECTIONS; i++) { - if (dnbd3_is_sock_alive(dev->socks[i])) { - if (index == sock_alive) { - sock = &dev->socks[i]; - } - sock_alive++; - } - } + if (!(sock->server && sock->sock && !sock->panic)) { +// warn_dev(dev, "attempted send on invalid socket %d", index); + msleep(SOCKET_TIMEOUT_CLIENT_DATA * 1000); - if (!sock) { // TODO let the mq queues be the same number as NUMBER_CONNECTIONS than handle them - warn_dev(dev, "index is %d but no socket was found", index); - dev_err_ratelimited(disk_to_dev(dev->disk), "attempted send on invalid socket\n"); - if (sock_alive > 0) { - blk_mq_update_nr_hw_queues(&dev->tag_set, sock_alive); - dnbd3_requeue_cmd(cmd); - ret = 0; - goto out; - } - if (first_try) { - debug_dev(dev, "no socket found, going to sleep"); - msleep(SOCKET_TIMEOUT_CLIENT_DATA * 1000); - first_try = false; - goto again; - } - error_dev(dev, "failed to find a socket, end request"); - blk_mq_end_request(req, BLK_STS_IOERR); - return -EINVAL; + dnbd3_requeue_cmd(cmd); + ret = 0; + goto out; } - cmd->status = BLK_STS_OK; mutex_lock(&sock->tx_lock); @@ -164,7 +141,7 @@ static blk_status_t dnbd3_queue_rq(struct blk_mq_hw_ctx *hctx, int ret; mutex_lock(&cmd->lock); - cmd->requed = false; +// cmd->requed = false; ret = dnbd3_handle_cmd(cmd, hctx->queue_num); if (ret < 0) { @@ -290,7 +267,6 @@ static int dnbd3_ioctl(struct block_device *bdev, fmode_t mode, } } - mutex_lock(&dev->device_lock); switch (cmd) { case IOCTL_OPEN: debug_dev(dev, "ioctl open"); @@ -301,13 +277,14 @@ static int dnbd3_ioctl(struct block_device *bdev, fmode_t mode, } else if (msg == NULL) { result = -EINVAL; } else { + mutex_lock(&dev->device_lock); if (sizeof(msg->host) != sizeof(dev->initial_server.host)) { warn_dev(dev, "odd size bug#1 triggered in ioctl"); } memcpy(&dev->initial_server.host, &msg->host, sizeof(msg->host)); dev->initial_server.failures = 0; - dnbd3_set_rtt_unreachable(&dev->initial_server) + dnbd3_set_rtt_unknown(&dev->initial_server) dev->imgname = imgname; dev->rid = msg->rid; dev->use_server_provided_alts = @@ -320,6 +297,9 @@ static int dnbd3_ioctl(struct block_device *bdev, fmode_t mode, * NUMBER_SERVERS); memcpy(dev->alt_servers, &dev->initial_server, sizeof(dev->alt_servers[0])); + + mutex_unlock(&dev->device_lock); + result = dnbd3_net_connect(dev); imgname = NULL; } @@ -352,12 +332,14 @@ static int dnbd3_ioctl(struct block_device *bdev, fmode_t mode, } else if (msg == NULL) { result = -EINVAL; } else { + mutex_lock(&dev->device_lock); memcpy(&dev->new_servers[dev->new_servers_num].host, &msg->host, sizeof(msg->host)); /* 0 = ADD, 1 = REM */ dev->new_servers[dev->new_servers_num].failures = (cmd == IOCTL_ADD_SRV ? 0 : 1); - ++dev->new_servers_num; + dev->new_servers_num++; + mutex_unlock(&dev->device_lock); result = 0; } break; @@ -372,7 +354,6 @@ static int dnbd3_ioctl(struct block_device *bdev, fmode_t mode, result = -EIO; break; } - mutex_unlock(&dev->device_lock); error: if (msg) kfree(msg); if (imgname) kfree(imgname); @@ -435,7 +416,7 @@ int dnbd3_add_device(struct dnbd3_device *dev, int minor) dev->disk = disk; dev->tag_set.ops = &dnbd3_mq_ops; /* this can be changed later with blk_mq_update_nr_hw_queues() */ - dev->tag_set.nr_hw_queues = 1; + dev->tag_set.nr_hw_queues = NUMBER_CONNECTIONS; dev->tag_set.queue_depth = 128; dev->tag_set.numa_node = NUMA_NO_NODE; dev->tag_set.cmd_size = sizeof(struct dnbd3_cmd); diff --git a/src/kernel/dnbd3.h b/src/kernel/dnbd3.h index 2c3ec39..8fabc35 100644 --- a/src/kernel/dnbd3.h +++ b/src/kernel/dnbd3.h @@ -39,6 +39,9 @@ + +#define RTT_FACTOR(rtt) (((rtt) * 3) / 2) + /** * limit to which the other connected servers are only allowed to be that worser * then the best rtt @@ -72,6 +75,7 @@ extern struct workqueue_struct *dnbd3_wq; struct dnbd3_server { dnbd3_host_t host; uint64_t rtts[4]; + uint64_t avg_rtt; uint16_t protocol_version; uint16_t failures; // TODO failures runterzaehlen wenn wieder verbindung moeglich, seltener messen bei hohem failure count @@ -208,7 +212,14 @@ struct dnbd3_cmd { #define dnbd3_set_rtt_unreachable(server) \ (server)->rtts[0] = (server)->rtts[1] = (server)->rtts[2] \ - = (server)->rtts[3] = RTT_UNREACHABLE; + = (server)->rtts[3] = (server)->avg_rtt = \ + RTT_UNREACHABLE; + + +#define dnbd3_set_rtt_unknown(server) \ + (server)->rtts[0] = (server)->rtts[1] = (server)->rtts[2] \ + = (server)->rtts[3] = (server)->avg_rtt = \ + RTT_UNREACHABLE / 2; /** diff --git a/src/kernel/net.c b/src/kernel/net.c index 5478e95..35c49b5 100644 --- a/src/kernel/net.c +++ b/src/kernel/net.c @@ -22,6 +22,7 @@ #include <net/sock.h> #include <linux/wait.h> +#include <linux/sort.h> #include "net.h" #include "utils.h" @@ -65,46 +66,11 @@ (h).msg_flags = MSG_WAITALL | MSG_NOSIGNAL; \ } while (0) - -#if NUMBER_CONNECTIONS == 1 -#define dnbd3_pl_socket_connect(dev, server) \ - dnbd3_socket_connect(dev, server) - -#define dnbd3_pl_socket_disconnect(sock) \ - dnbd3_socket_disconnect(sock) - - -#else -#define dnbd3_pl_socket_connect(sock, server) \ - do {\ - int i; \ - for (i = 0 < NUMBER_PARALLEL_CONNECTIONS; i++) { \ - dnbd3_socket_connect(dev, server); \ - dnbd3_socket_connect((sock) + \ - (i * sizeof(struct dnbd3_sock)), \ - (server)); \ - } \ - } while (0) - - -#define dnbd3_pl_socket_disconnect(sock) \ - do { \ - int i; \ - for (i = 0 < NUMBER_PARALLEL_CONNECTIONS; i++) { \ - dnbd3_socket_disconnect((sock) + \ - (i * sizeof(struct dnbd3_sock))); \ - } \ - } while (0) - -#endif - -static int dnbd3_server_connect(struct dnbd3_device *dev, - struct dnbd3_server *server); +static int __dnbd3_socket_connect(struct dnbd3_sock *sock, + struct dnbd3_server * server); static int dnbd3_socket_connect(struct dnbd3_sock *sock, struct dnbd3_server * server); static int dnbd3_socket_disconnect(struct dnbd3_sock *sock); -static int dnbd3_server_disconnect(struct dnbd3_device *dev, - struct dnbd3_server *server); /* @@ -216,8 +182,8 @@ int dnbd3_send_request(struct dnbd3_sock *sock, struct request *req, goto error; } - sock->pending = NULL; error: + sock->pending = NULL; return result; } @@ -738,176 +704,107 @@ static void dnbd3_keepalive_worker(struct work_struct *work) } /** - * dnbd3_find_best_alt_server - find best alternative server - * @dev: the device where to search for alternative servers - * - * searches for an alternative server which has an rtt better than RTT_THRESOLD - * of the best connected server + * dnbd3_compare_servers - comparator for the server + * @lhs: left hand sign + * @rhs: right hand sign */ -static struct dnbd3_server *dnbd3_find_best_alt_server(struct dnbd3_device *dev) -{ - int i, j; - uint64_t rtt = 0; - uint64_t best_rtt = RTT_UNREACHABLE; - uint64_t current_best_rtt = RTT_UNREACHABLE; - struct dnbd3_server *best_server = NULL; - struct dnbd3_server *server = NULL; - - for (i = 0; i < NUMBER_CONNECTIONS; i++) { - if (dnbd3_is_sock_alive(dev->socks[i])) { - rtt = dnbd3_avg_rtt(dev->socks[i].server); - if (rtt <= current_best_rtt) { - current_best_rtt = rtt; - } - } +static int dnbd3_compare_servers(const void *lhs, const void *rhs) { + uint64_t l, r; + struct dnbd3_server *lhs_server = *((struct dnbd3_server **) lhs); + struct dnbd3_server *rhs_server = *((struct dnbd3_server **) rhs); + + l = lhs_server->host.type != 0 ? lhs_server->avg_rtt + : RTT_UNREACHABLE + 1; + r = rhs_server->host.type != 0 ? rhs_server->avg_rtt + : RTT_UNREACHABLE + 1; + if (l < r) { + return -1; + } else if (l > r) { + return 1; } - - best_rtt = RTT_THRESOULD_LIMIT(current_best_rtt); - debug_dev(dev, "best connected rtt is %llu, searching for rtt better than %llu", - current_best_rtt, best_rtt); - - for (i = 0; i < NUMBER_SERVERS; i++) { - if (dev->alt_servers[i].host.type != 0) { - rtt = dnbd3_avg_rtt(&dev->alt_servers[i]); - if (rtt <= best_rtt) { - server = &dev->alt_servers[i]; - /* check if already connected */ - for (j = 0; j < NUMBER_CONNECTIONS; j++) { - if (server == dev->socks[j].server) { - server = NULL; - break; - } - } - if (server) { - best_server = server; - best_rtt = rtt; - } - } - } - } - if (best_server) { - debug_server(dev, best_server, "found best alt server with rtt %llu", - best_rtt); - } else { - debug_dev(dev, "did not find any alternative server"); - } - return best_server; + return 0; } /** - * dnbd3_better_rtt - checks if the rtt is better - * @new_server: the server to check - * @existing_server: current server + * dnbd3_sort_server - sort the alt server according to their avg rtt + * @dev: the dndb3 device * - * checks if the rtt is better than RTT_THRESHOLD_FACTOR + * the returned array has to be freed with kfree */ -static bool dnbd3_better_rtt(struct dnbd3_server *new_server, - struct dnbd3_server *existing_server) { - uint64_t new_rtt = dnbd3_avg_rtt(new_server); - uint64_t existing_rtt = dnbd3_avg_rtt(existing_server); +static struct dnbd3_server **dnbd3_sort_server(struct dnbd3_device *dev) { + int i; + struct dnbd3_server **sorted_servers = kmalloc(NUMBER_SERVERS * sizeof(struct dnbd3_device *), GFP_KERNEL); + if (!sorted_servers) { + return NULL; + } - if (new_rtt < RTT_THRESHOLD_FACTOR(existing_rtt)) { - return true; + for (i = 0; i < NUMBER_SERVERS; i++) { + sorted_servers[i] = &dev->alt_servers[i]; } - return false; + sort(sorted_servers, NUMBER_SERVERS, sizeof(struct dnbd3_device *), + &dnbd3_compare_servers, NULL); + + return sorted_servers; } /** - * dnbd3_adjust_connections - adjust the connections of the device - * @dev: the device + * dnbd3_adjust_connections - create a connection plan and connect + * @dev: the dnbd3 device * - * 1. connect empty sockets if best alternative server is found - * 2. replace slow socket with better server if available - * 3. remove socket if one is slow + * 1. sort the alt server after the avg rtt + * 2. create a connection plan + * 3. connect the plan */ -static void dnbd3_adjust_connections(struct dnbd3_device *dev) { - int i; - int sock_alive = 0; - uint64_t rtt; - uint64_t best_rtt = RTT_UNREACHABLE; - struct dnbd3_server *server, *existing_server; +static int dnbd3_adjust_connections(struct dnbd3_device *dev) { + int i, j, fallback; + struct dnbd3_server *plan[NUMBER_CONNECTIONS]; + struct dnbd3_server **servers = dnbd3_sort_server(dev); - /* connect empty sockets */ //TODO check if panic worker connects at the same time, than we would have a problem - sock_alive = 0; - for (i = 0; i < NUMBER_CONNECTIONS; i++) { - if (!dnbd3_is_sock_alive(dev->socks[i])) { - server = dnbd3_find_best_alt_server(dev); - if (server) { - if (dnbd3_server_connect(dev, server) == 0) { - sock_alive++; + if (servers && servers[0]->host.type != 0) { + mutex_lock(&dev->device_lock); + plan[0] = servers[0]; + fallback = 0; + j = 1; + debug_dev(dev, "connection plan:"); + debug_server(dev, plan[0], "server 0 with rtt %llu:", + plan[0]->avg_rtt); + for (i = 1; i < NUMBER_CONNECTIONS; i++) { + if (servers[j]->host.type != 0) { + if (RTT_FACTOR(plan[i - 1]->avg_rtt) > + servers[j]->avg_rtt) { + plan[i] = servers[j]; + j++; } else { - warn_server(dev, server, "failed to connect"); + plan[i] = plan[fallback]; + fallback++; } + } else { + plan[i] = plan[fallback]; + fallback++; } - } else { - sock_alive++; + debug_server(dev, plan[i], "server %d with rtt %llu:", + i, plan[i]->avg_rtt); } - } + kfree(servers); - /* replace socket with better server */ - if (sock_alive == NUMBER_CONNECTIONS) { for (i = 0; i < NUMBER_CONNECTIONS; i++) { - if (dnbd3_is_sock_alive(dev->socks[i])) { - server = dnbd3_find_best_alt_server(dev); - existing_server = dev->socks[i].server; - if (server && dnbd3_better_rtt(server, dev->socks[i].server)) { + if (plan[i] != dev->socks[i].server) { + if (dnbd3_is_sock_alive(dev->socks[i])) { dnbd3_socket_disconnect(&dev->socks[i]); - - /* if it fails reconnect to existing */ - if (dnbd3_server_connect(dev, server) != 0) { - warn_server(dev, server, "failed to connect"); - dnbd3_server_connect(dev, existing_server); - } - } - } - } - } - - /* remove a socket if it is much slower than the others */ - if (sock_alive > 1) { - for (i = 0; i < NUMBER_CONNECTIONS; i++) { - if (dnbd3_is_sock_alive(dev->socks[i])) { - rtt = dnbd3_avg_rtt(dev->socks[i].server); - if (rtt <= best_rtt) { - best_rtt = rtt; } + dnbd3_socket_connect(&dev->socks[i], plan[i]); } } - for (i = 0; i < NUMBER_CONNECTIONS; i++) { - if (dnbd3_is_sock_alive(dev->socks[i])) { - rtt = dnbd3_avg_rtt(dev->socks[i].server); - if (rtt > RTT_THRESOULD_LIMIT(best_rtt)) { - info_sock(&dev->socks[i], "removing connection with rtt %llu", rtt); - dnbd3_socket_disconnect(&dev->socks[i]); - sock_alive--; - } - } + mutex_unlock(&dev->device_lock); + return 0; + } else { /* there is nothing to connect */ + if (servers) { + kfree(servers); } + return -ENONET; } - info_dev(dev, "connected to %d/%d sockets", sock_alive, NUMBER_CONNECTIONS); -} -/** - * dnbd3_panic_connect - connect to the first available server - * @dev: the device - */ -static int dnbd3_panic_connect(struct dnbd3_device *dev) { - int result, i; - result = dnbd3_server_connect(dev, &dev->initial_server); - if (result) { - for (i = 0; i < NUMBER_SERVERS; i++) { - if (dev->alt_servers[i].host.type != 0) { - result = dnbd3_server_connect(dev, - &dev->alt_servers[i]); - if (!result) { - info_server(dev, &dev->alt_servers[i], "found server to connect to"); - break; - } - } - } - } - return result; } /** @@ -921,47 +818,27 @@ static int dnbd3_panic_connect(struct dnbd3_device *dev) { static void dnbd3_panic_worker(struct work_struct *work) { struct dnbd3_device *dev; - struct dnbd3_sock *panicked_sock = NULL; - struct dnbd3_server *new_server, *panicked_server; + bool panic = false; int i; int sock_alive = 0; dev = container_of(work, struct dnbd3_device, panic_worker); for (i = 0; i < NUMBER_CONNECTIONS; i++) { if (dev->socks[i].panic) { - panicked_sock = &dev->socks[i]; + panic = true; + dnbd3_set_rtt_unreachable(dev->socks[i].server); + dnbd3_socket_disconnect(&dev->socks[i]); + } else if (dnbd3_is_sock_alive(dev->socks[i])) { sock_alive++; } } - if (panicked_sock) { - warn_sock(panicked_sock, "panicked, connections still alive %d", - sock_alive); - panicked_server = panicked_sock->server; - new_server = dnbd3_find_best_alt_server(dev); - dnbd3_socket_disconnect(panicked_sock); - dnbd3_set_rtt_unreachable(panicked_server); + if (panic) { + warn_dev(dev, "panicked, connections still alive %d", + sock_alive); - if (new_server != NULL && new_server != panicked_server) { - info_server(dev, new_server, "found replacement"); - if (!dnbd3_server_connect(dev, new_server)) { - sock_alive++; - } - } else if (sock_alive > 0) { - info_dev(dev, "found no replacement server but still connected to %d servers", - sock_alive); - } - } - - if (sock_alive == 0) { - warn_dev(dev, "did not find a good server, trying to connect to any available server"); - - if (dnbd3_panic_connect(dev)) { - error_dev(dev, "could not connect to any server"); - } else { - info_dev(dev, "found server to connect to"); - } + dnbd3_adjust_connections(dev); } } @@ -985,7 +862,7 @@ static int dnbd3_meassure_rtt(struct dnbd3_device *dev, .server = server }; - result = dnbd3_socket_connect(&sock, server); + result = __dnbd3_socket_connect(&sock, server); if (result) { error_sock(&sock, "socket connect failed in rtt measurement"); goto error; @@ -1044,6 +921,7 @@ static int dnbd3_meassure_rtt(struct dnbd3_device *dev, error: sock.server->rtts[dev->discovery_count % 4] = rtt; + sock.server->avg_rtt = dnbd3_avg_rtt(sock.server); if (result <= 0) { server->failures++; } @@ -1092,8 +970,9 @@ static void dnbd3_merge_new_server(struct dnbd3_device *dev, if (new_server->failures == 1) { /* remove is requested */ info_server(dev, new_server, "remove server is requested"); - dnbd3_server_disconnect(dev, existing_server); + // adjust connection will remove it later existing_server->host.type = 0; + dnbd3_set_rtt_unreachable(existing_server); } // existing_server->failures = 0; // reset failure count return; @@ -1110,9 +989,10 @@ static void dnbd3_merge_new_server(struct dnbd3_device *dev, info_server(dev, free_server, "got new alternative server"); free_server->failures = 0; free_server->protocol_version = 0; - dnbd3_set_rtt_unreachable(free_server); + dnbd3_set_rtt_unknown(free_server); } + /** * dnbd3_discovery_worker - handle discovery * @work: the work used to get the dndb3_device @@ -1168,11 +1048,11 @@ static void dnbd3_discovery_worker(struct work_struct *work) /** - * dnbd3_socket_connect - connect a socket to a server + * __dnbd3_socket_connect - internal connect a socket to a server * @sock: the socket to connect * @server: the server */ -static int dnbd3_socket_connect(struct dnbd3_sock *sock, +static int __dnbd3_socket_connect(struct dnbd3_sock *sock, struct dnbd3_server *server) { int result = 0; @@ -1237,44 +1117,27 @@ error: } /** - * dnbd3_server_connect - connect a server to a device - * @dev: the device + * dnbd3_socket_connect - connect a socket to a server + * @sock: the socket * @server: the server to connect * - * 1. connects the server to a free socket if available + * 1. connects the server to the socket * 2. select the image * 3. start receiver_worker and keepalive_worker - * 4. if it is the first connection start timer, panic_worker and - * keepalive_worker - * 5. update the mq queues to the number of sockets alive */ -static int dnbd3_server_connect(struct dnbd3_device *dev, +static int dnbd3_socket_connect(struct dnbd3_sock *sock, struct dnbd3_server *server) { - int i; - int sock_alive = 0; int result = -EIO; dnbd3_reply_t reply; - struct dnbd3_sock *sock = NULL; - for (i = 0; i < NUMBER_CONNECTIONS; i++) { - if (!dnbd3_is_sock_alive(dev->socks[i])) { - sock = &dev->socks[i]; - break; - } - } - if (sock == NULL) { - warn_server(dev, server, "could not connect to socket, to many connections"); - return -EIO; - } + struct dnbd3_device *dev = sock->device; sock->server = server; - debug_sock(sock, "socket connect"); - mutex_init(&sock->tx_lock); mutex_lock(&sock->tx_lock); - result = dnbd3_socket_connect(sock, server); + result = __dnbd3_socket_connect(sock, server); if (result) { error_sock(sock, "connection to socket failed"); result = -EIO; @@ -1294,6 +1157,7 @@ static int dnbd3_server_connect(struct dnbd3_device *dev, result = dnbd3_send_request_cmd(sock, CMD_SELECT_IMAGE); if (result <= 0) { + error_sock(sock, "connection to image %s failed", dev->imgname); result = -EIO; goto error; @@ -1328,23 +1192,7 @@ static int dnbd3_server_connect(struct dnbd3_device *dev, INIT_WORK(&sock->keepalive_worker, dnbd3_keepalive_worker); - for (i = 0; i < NUMBER_CONNECTIONS; i++) { - if (dev->socks[i].sock && dev->socks[i].server) { - sock_alive++; - } - } - /* if first socket to connect, start timer and workers */ - if (sock_alive == 1) { - debug_sock(sock, "first connection to server, starting workers"); - INIT_WORK(&dev->discovery_worker, dnbd3_discovery_worker); - INIT_WORK(&dev->panic_worker, dnbd3_panic_worker); - timer_setup(&dev->timer, dnbd3_timer, 0); - dev->timer.expires = jiffies + HZ; - add_timer(&dev->timer); - } - blk_mq_update_nr_hw_queues(&dev->tag_set, sock_alive); - - /* request alternative servers receiver will handle this */ + /* TODO not on every connect? request alternative servers receiver will handle this */ if (dnbd3_send_request_cmd(sock, CMD_GET_SERVERS) <= 0) { error_sock(sock, "failed to get servers in discovery"); } @@ -1352,6 +1200,7 @@ static int dnbd3_server_connect(struct dnbd3_device *dev, return 0; error: server->failures++; + sock->panic = true; if (sock->sock) { kernel_sock_shutdown(sock->sock, SHUT_RDWR); cancel_work_sync(&sock->receive_worker); @@ -1362,41 +1211,13 @@ error: return result; } -static void dnbd3_clear_req(struct request *req, void *data, bool reserved) -{ - struct dnbd3_cmd *cmd = blk_mq_rq_to_pdu(req); - - cmd->status = BLK_STS_IOERR; - blk_mq_complete_request(req); -} /** - * dnbd3_socket_disconnect - disconnect a socket or server + * dnbd3_socket_disconnect - disconnect a socket * @sock: the socket to disconnect - * - * 1. update nr of mq queues - * 2. if last socket remove timer - * 3. disconnect socket */ static int dnbd3_socket_disconnect(struct dnbd3_sock *sock) { - int i; - struct dnbd3_device *dev = sock->device; - int sock_alive = 0; - for (i = 0; i < NUMBER_CONNECTIONS; i++) { - if (dnbd3_is_sock_alive(dev->socks[i])) { - sock_alive++; - } - } - if (sock_alive <= 1) { - info_sock(sock, "shutting down last socket and stopping timer"); - del_timer_sync(&dev->timer); - /* - * do not wait for discovery and panic worker as they may have - * called this method - */ - - } cancel_work_sync(&sock->keepalive_worker); debug_sock(sock, "socket disconnect"); @@ -1425,50 +1246,18 @@ static int dnbd3_socket_disconnect(struct dnbd3_sock *sock) sock_release(sock->sock); sock->sock = NULL; } - debug_sock(sock, "update nr hw queues to %d", sock_alive -1); -// blk_mq_quiesce_queue(dev->disk->queue); -// debug_sock(sock, "busy iter"); -// blk_mq_tagset_busy_iter(&dev->tag_set, dnbd3_clear_req, NULL); -// debug_sock(sock, "update"); - //TODO can we just stop the failed queue? - blk_mq_update_nr_hw_queues(&dev->tag_set, sock_alive - 1); -// debug_sock(sock, "unquisco"); -// blk_mq_unquiesce_queue(dev->disk->queue); -// debug_sock(sock, "donee"); sock->server = NULL; sock->panic = false; return 0; } -/** - * dnbd3_server_disconnect - disconnect a server from a socket - * @dev: the device - * @server: the server to disconnect - */ -static int dnbd3_server_disconnect(struct dnbd3_device *dev, - struct dnbd3_server *server) -{ - int i; - struct dnbd3_sock *sock = NULL; - for (i = 0; i < NUMBER_CONNECTIONS; i++) { - if (dev->socks[i].server == server) { - sock = &dev->socks[i]; - } - } - if (!sock || !sock->sock) { - warn_dev(dev, "could not find socket to disconnect"); - return -EIO; - } - return dnbd3_socket_disconnect(sock); -} - /** * dnbd3_net_connect - connect device * @dev: the device to connect * - * dnbd3_device.alt_servers[0] must set + * dnbd3_device.alt_servers[0] must be set */ int dnbd3_net_connect(struct dnbd3_device *dev) { @@ -1479,14 +1268,27 @@ int dnbd3_net_connect(struct dnbd3_device *dev) return -ENONET; } - // alt_server[0] is the initial server - result = dnbd3_server_connect(dev, &dev->alt_servers[0]); + result = dnbd3_adjust_connections(dev); if (result) { error_dev(dev, "failed to connect to initial server"); - result = -ENOENT; - dev->imgname = NULL; - dev->socks[0].server = NULL; + return -ENOENT; } + + debug_dev(dev, "connected, starting workers"); + INIT_WORK(&dev->discovery_worker, dnbd3_discovery_worker); + INIT_WORK(&dev->panic_worker, dnbd3_panic_worker); + timer_setup(&dev->timer, dnbd3_timer, 0); + dev->timer.expires = jiffies + HZ; + add_timer(&dev->timer); + + // alt_server[0] is the initial server +// result = dnbd3_server_connect(dev, &dev->alt_servers[0]); +// if (result) { +// error_dev(dev, "failed to connect to initial server"); +// result = -ENOENT; +// dev->imgname = NULL; +// dev->socks[0].server = NULL; +// } return result; } diff --git a/src/kernel/sysfs.c b/src/kernel/sysfs.c index 1588dfb..490ffbd 100644 --- a/src/kernel/sysfs.c +++ b/src/kernel/sysfs.c @@ -46,18 +46,18 @@ ssize_t show_connected_servers(char *buf, struct dnbd3_device *dev) if (dev->socks[i].sock) { if (dev->socks[i].server->host.type == HOST_IP4) { ret = MIN(snprintf(buf, size, "%pI4,%d,%llu,%d\n", - dev->socks[i].server->host.addr, - (int)ntohs(dev->socks[i].server->host.port), - (unsigned long long)((dev->socks[i].server->rtts[0] + dev->socks[i].server->rtts[1] + dev->socks[i].server->rtts[2] + dev->socks[i].server->rtts[3]) / 4), - (int)dev->socks[i].server->failures) + dev->socks[i].server->host.addr, + (int)ntohs(dev->socks[i].server->host.port), + dev->socks[i].server->avg_rtt, + (int)dev->socks[i].server->failures) , size); } else { ret = MIN(snprintf(buf, size, "%pI6,%d,%llu,%d\n", - dev->socks[i].server->host.addr, - (int)ntohs(dev->socks[i].server->host.port), - (unsigned long long)((dev->socks[i].server->rtts[0] + dev->socks[i].server->rtts[1] + dev->socks[i].server->rtts[2] + dev->socks[i].server->rtts[3]) / 4), - (int)dev->socks[i].server->failures) + dev->socks[i].server->host.addr, + (int)ntohs(dev->socks[i].server->host.port), + dev->socks[i].server->avg_rtt, + (int)dev->socks[i].server->failures) , size); } @@ -81,9 +81,10 @@ ssize_t show_connected_servers(char *buf, struct dnbd3_device *dev) ssize_t show_alt_server_num(char *buf, struct dnbd3_device *dev) { int i, num = 0; - for (i = 0; i < NUMBER_SERVERS; ++i) - { - if (dev->alt_servers[i].host.type) ++num; + for (i = 0; i < NUMBER_SERVERS; i++) { + if (dev->alt_servers[i].host.type) { + num++; + } } return MIN(snprintf(buf, PAGE_SIZE, "%d\n", num), PAGE_SIZE); } @@ -91,28 +92,27 @@ ssize_t show_alt_server_num(char *buf, struct dnbd3_device *dev) ssize_t show_alt_servers(char *buf, struct dnbd3_device *dev) { int i, size = PAGE_SIZE, ret; - for (i = 0; i < NUMBER_SERVERS; ++i) - { - if (dev->alt_servers[i].host.type == HOST_IP4) + for (i = 0; i < NUMBER_SERVERS; ++i) { + if (dev->alt_servers[i].host.type == HOST_IP4) { ret = MIN(snprintf(buf, size, "%pI4,%d,%llu,%d\n", dev->alt_servers[i].host.addr, (int)ntohs(dev->alt_servers[i].host.port), - (unsigned long long)((dev->alt_servers[i].rtts[0] + dev->alt_servers[i].rtts[1] + dev->alt_servers[i].rtts[2] + dev->alt_servers[i].rtts[3]) / 4), + dev->alt_servers[i].avg_rtt, (int)dev->alt_servers[i].failures) , size); - else if (dev->alt_servers[i].host.type == HOST_IP6) + } else if (dev->alt_servers[i].host.type == HOST_IP6) { ret = MIN(snprintf(buf, size, "%pI6,%d,%llu,%d\n", dev->alt_servers[i].host.addr, (int)ntohs(dev->alt_servers[i].host.port), - (unsigned long long)((dev->alt_servers[i].rtts[0] + dev->alt_servers[i].rtts[1] + dev->alt_servers[i].rtts[2] + dev->alt_servers[i].rtts[3]) / 4), + dev->alt_servers[i].avg_rtt, (int)dev->alt_servers[i].failures) , size); - else + } else { continue; + } size -= ret; buf += ret; - if (size <= 0) - { + if (size <= 0) { size = 0; break; } |