summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
-rw-r--r--src/kernel/core.c61
-rw-r--r--src/kernel/dnbd3.h13
-rw-r--r--src/kernel/net.c442
-rw-r--r--src/kernel/sysfs.c40
4 files changed, 175 insertions, 381 deletions
diff --git a/src/kernel/core.c b/src/kernel/core.c
index 8bb23d9..6f0dab4 100644
--- a/src/kernel/core.c
+++ b/src/kernel/core.c
@@ -72,6 +72,8 @@ static void dnbd3_requeue_cmd(struct dnbd3_cmd *cmd)
if (!cmd->requed) {
cmd->requed = true;
blk_mq_requeue_request(req, true);
+ } else {
+ blk_mq_end_request(req, BLK_STS_IOERR);
}
}
@@ -84,47 +86,22 @@ static int dnbd3_handle_cmd(struct dnbd3_cmd *cmd, int index)
{
struct request *req = blk_mq_rq_from_pdu(cmd);
struct dnbd3_device *dev = cmd->dnbd3;
- struct dnbd3_sock *sock = NULL;
- bool first_try = true;
+ struct dnbd3_sock *sock = &dev->socks[index];
int ret = -1;
- int i;
- int sock_alive = 0;
- debug_dev(dev, "handle request at position %lu, size %d, index %d",
- blk_rq_pos(req), blk_rq_bytes(req), index);
+// debug_dev(dev, "handle request at position %lu, size %d, index %d",
+// blk_rq_pos(req), blk_rq_bytes(req), index);
-again:
- for (i = 0; i < NUMBER_CONNECTIONS; i++) {
- if (dnbd3_is_sock_alive(dev->socks[i])) {
- if (index == sock_alive) {
- sock = &dev->socks[i];
- }
- sock_alive++;
- }
- }
+ if (!(sock->server && sock->sock && !sock->panic)) {
+// warn_dev(dev, "attempted send on invalid socket %d", index);
+ msleep(SOCKET_TIMEOUT_CLIENT_DATA * 1000);
- if (!sock) { // TODO let the mq queues be the same number as NUMBER_CONNECTIONS than handle them
- warn_dev(dev, "index is %d but no socket was found", index);
- dev_err_ratelimited(disk_to_dev(dev->disk), "attempted send on invalid socket\n");
- if (sock_alive > 0) {
- blk_mq_update_nr_hw_queues(&dev->tag_set, sock_alive);
- dnbd3_requeue_cmd(cmd);
- ret = 0;
- goto out;
- }
- if (first_try) {
- debug_dev(dev, "no socket found, going to sleep");
- msleep(SOCKET_TIMEOUT_CLIENT_DATA * 1000);
- first_try = false;
- goto again;
- }
- error_dev(dev, "failed to find a socket, end request");
- blk_mq_end_request(req, BLK_STS_IOERR);
- return -EINVAL;
+ dnbd3_requeue_cmd(cmd);
+ ret = 0;
+ goto out;
}
-
cmd->status = BLK_STS_OK;
mutex_lock(&sock->tx_lock);
@@ -164,7 +141,7 @@ static blk_status_t dnbd3_queue_rq(struct blk_mq_hw_ctx *hctx,
int ret;
mutex_lock(&cmd->lock);
- cmd->requed = false;
+// cmd->requed = false;
ret = dnbd3_handle_cmd(cmd, hctx->queue_num);
if (ret < 0) {
@@ -290,7 +267,6 @@ static int dnbd3_ioctl(struct block_device *bdev, fmode_t mode,
}
}
- mutex_lock(&dev->device_lock);
switch (cmd) {
case IOCTL_OPEN:
debug_dev(dev, "ioctl open");
@@ -301,13 +277,14 @@ static int dnbd3_ioctl(struct block_device *bdev, fmode_t mode,
} else if (msg == NULL) {
result = -EINVAL;
} else {
+ mutex_lock(&dev->device_lock);
if (sizeof(msg->host) != sizeof(dev->initial_server.host)) {
warn_dev(dev, "odd size bug#1 triggered in ioctl");
}
memcpy(&dev->initial_server.host, &msg->host,
sizeof(msg->host));
dev->initial_server.failures = 0;
- dnbd3_set_rtt_unreachable(&dev->initial_server)
+ dnbd3_set_rtt_unknown(&dev->initial_server)
dev->imgname = imgname;
dev->rid = msg->rid;
dev->use_server_provided_alts =
@@ -320,6 +297,9 @@ static int dnbd3_ioctl(struct block_device *bdev, fmode_t mode,
* NUMBER_SERVERS);
memcpy(dev->alt_servers, &dev->initial_server,
sizeof(dev->alt_servers[0]));
+
+ mutex_unlock(&dev->device_lock);
+
result = dnbd3_net_connect(dev);
imgname = NULL;
}
@@ -352,12 +332,14 @@ static int dnbd3_ioctl(struct block_device *bdev, fmode_t mode,
} else if (msg == NULL) {
result = -EINVAL;
} else {
+ mutex_lock(&dev->device_lock);
memcpy(&dev->new_servers[dev->new_servers_num].host,
&msg->host, sizeof(msg->host));
/* 0 = ADD, 1 = REM */
dev->new_servers[dev->new_servers_num].failures =
(cmd == IOCTL_ADD_SRV ? 0 : 1);
- ++dev->new_servers_num;
+ dev->new_servers_num++;
+ mutex_unlock(&dev->device_lock);
result = 0;
}
break;
@@ -372,7 +354,6 @@ static int dnbd3_ioctl(struct block_device *bdev, fmode_t mode,
result = -EIO;
break;
}
- mutex_unlock(&dev->device_lock);
error:
if (msg) kfree(msg);
if (imgname) kfree(imgname);
@@ -435,7 +416,7 @@ int dnbd3_add_device(struct dnbd3_device *dev, int minor)
dev->disk = disk;
dev->tag_set.ops = &dnbd3_mq_ops;
/* this can be changed later with blk_mq_update_nr_hw_queues() */
- dev->tag_set.nr_hw_queues = 1;
+ dev->tag_set.nr_hw_queues = NUMBER_CONNECTIONS;
dev->tag_set.queue_depth = 128;
dev->tag_set.numa_node = NUMA_NO_NODE;
dev->tag_set.cmd_size = sizeof(struct dnbd3_cmd);
diff --git a/src/kernel/dnbd3.h b/src/kernel/dnbd3.h
index 2c3ec39..8fabc35 100644
--- a/src/kernel/dnbd3.h
+++ b/src/kernel/dnbd3.h
@@ -39,6 +39,9 @@
+
+#define RTT_FACTOR(rtt) (((rtt) * 3) / 2)
+
/**
* limit to which the other connected servers are only allowed to be that worser
* then the best rtt
@@ -72,6 +75,7 @@ extern struct workqueue_struct *dnbd3_wq;
struct dnbd3_server {
dnbd3_host_t host;
uint64_t rtts[4];
+ uint64_t avg_rtt;
uint16_t protocol_version;
uint16_t failures; // TODO failures runterzaehlen wenn wieder verbindung moeglich, seltener messen bei hohem failure count
@@ -208,7 +212,14 @@ struct dnbd3_cmd {
#define dnbd3_set_rtt_unreachable(server) \
(server)->rtts[0] = (server)->rtts[1] = (server)->rtts[2] \
- = (server)->rtts[3] = RTT_UNREACHABLE;
+ = (server)->rtts[3] = (server)->avg_rtt = \
+ RTT_UNREACHABLE;
+
+
+#define dnbd3_set_rtt_unknown(server) \
+ (server)->rtts[0] = (server)->rtts[1] = (server)->rtts[2] \
+ = (server)->rtts[3] = (server)->avg_rtt = \
+ RTT_UNREACHABLE / 2;
/**
diff --git a/src/kernel/net.c b/src/kernel/net.c
index 5478e95..35c49b5 100644
--- a/src/kernel/net.c
+++ b/src/kernel/net.c
@@ -22,6 +22,7 @@
#include <net/sock.h>
#include <linux/wait.h>
+#include <linux/sort.h>
#include "net.h"
#include "utils.h"
@@ -65,46 +66,11 @@
(h).msg_flags = MSG_WAITALL | MSG_NOSIGNAL; \
} while (0)
-
-#if NUMBER_CONNECTIONS == 1
-#define dnbd3_pl_socket_connect(dev, server) \
- dnbd3_socket_connect(dev, server)
-
-#define dnbd3_pl_socket_disconnect(sock) \
- dnbd3_socket_disconnect(sock)
-
-
-#else
-#define dnbd3_pl_socket_connect(sock, server) \
- do {\
- int i; \
- for (i = 0 < NUMBER_PARALLEL_CONNECTIONS; i++) { \
- dnbd3_socket_connect(dev, server); \
- dnbd3_socket_connect((sock) + \
- (i * sizeof(struct dnbd3_sock)), \
- (server)); \
- } \
- } while (0)
-
-
-#define dnbd3_pl_socket_disconnect(sock) \
- do { \
- int i; \
- for (i = 0 < NUMBER_PARALLEL_CONNECTIONS; i++) { \
- dnbd3_socket_disconnect((sock) + \
- (i * sizeof(struct dnbd3_sock))); \
- } \
- } while (0)
-
-#endif
-
-static int dnbd3_server_connect(struct dnbd3_device *dev,
- struct dnbd3_server *server);
+static int __dnbd3_socket_connect(struct dnbd3_sock *sock,
+ struct dnbd3_server * server);
static int dnbd3_socket_connect(struct dnbd3_sock *sock,
struct dnbd3_server * server);
static int dnbd3_socket_disconnect(struct dnbd3_sock *sock);
-static int dnbd3_server_disconnect(struct dnbd3_device *dev,
- struct dnbd3_server *server);
/*
@@ -216,8 +182,8 @@ int dnbd3_send_request(struct dnbd3_sock *sock, struct request *req,
goto error;
}
- sock->pending = NULL;
error:
+ sock->pending = NULL;
return result;
}
@@ -738,176 +704,107 @@ static void dnbd3_keepalive_worker(struct work_struct *work)
}
/**
- * dnbd3_find_best_alt_server - find best alternative server
- * @dev: the device where to search for alternative servers
- *
- * searches for an alternative server which has an rtt better than RTT_THRESOLD
- * of the best connected server
+ * dnbd3_compare_servers - comparator for the server
+ * @lhs: left hand sign
+ * @rhs: right hand sign
*/
-static struct dnbd3_server *dnbd3_find_best_alt_server(struct dnbd3_device *dev)
-{
- int i, j;
- uint64_t rtt = 0;
- uint64_t best_rtt = RTT_UNREACHABLE;
- uint64_t current_best_rtt = RTT_UNREACHABLE;
- struct dnbd3_server *best_server = NULL;
- struct dnbd3_server *server = NULL;
-
- for (i = 0; i < NUMBER_CONNECTIONS; i++) {
- if (dnbd3_is_sock_alive(dev->socks[i])) {
- rtt = dnbd3_avg_rtt(dev->socks[i].server);
- if (rtt <= current_best_rtt) {
- current_best_rtt = rtt;
- }
- }
+static int dnbd3_compare_servers(const void *lhs, const void *rhs) {
+ uint64_t l, r;
+ struct dnbd3_server *lhs_server = *((struct dnbd3_server **) lhs);
+ struct dnbd3_server *rhs_server = *((struct dnbd3_server **) rhs);
+
+ l = lhs_server->host.type != 0 ? lhs_server->avg_rtt
+ : RTT_UNREACHABLE + 1;
+ r = rhs_server->host.type != 0 ? rhs_server->avg_rtt
+ : RTT_UNREACHABLE + 1;
+ if (l < r) {
+ return -1;
+ } else if (l > r) {
+ return 1;
}
-
- best_rtt = RTT_THRESOULD_LIMIT(current_best_rtt);
- debug_dev(dev, "best connected rtt is %llu, searching for rtt better than %llu",
- current_best_rtt, best_rtt);
-
- for (i = 0; i < NUMBER_SERVERS; i++) {
- if (dev->alt_servers[i].host.type != 0) {
- rtt = dnbd3_avg_rtt(&dev->alt_servers[i]);
- if (rtt <= best_rtt) {
- server = &dev->alt_servers[i];
- /* check if already connected */
- for (j = 0; j < NUMBER_CONNECTIONS; j++) {
- if (server == dev->socks[j].server) {
- server = NULL;
- break;
- }
- }
- if (server) {
- best_server = server;
- best_rtt = rtt;
- }
- }
- }
- }
- if (best_server) {
- debug_server(dev, best_server, "found best alt server with rtt %llu",
- best_rtt);
- } else {
- debug_dev(dev, "did not find any alternative server");
- }
- return best_server;
+ return 0;
}
/**
- * dnbd3_better_rtt - checks if the rtt is better
- * @new_server: the server to check
- * @existing_server: current server
+ * dnbd3_sort_server - sort the alt server according to their avg rtt
+ * @dev: the dndb3 device
*
- * checks if the rtt is better than RTT_THRESHOLD_FACTOR
+ * the returned array has to be freed with kfree
*/
-static bool dnbd3_better_rtt(struct dnbd3_server *new_server,
- struct dnbd3_server *existing_server) {
- uint64_t new_rtt = dnbd3_avg_rtt(new_server);
- uint64_t existing_rtt = dnbd3_avg_rtt(existing_server);
+static struct dnbd3_server **dnbd3_sort_server(struct dnbd3_device *dev) {
+ int i;
+ struct dnbd3_server **sorted_servers = kmalloc(NUMBER_SERVERS * sizeof(struct dnbd3_device *), GFP_KERNEL);
+ if (!sorted_servers) {
+ return NULL;
+ }
- if (new_rtt < RTT_THRESHOLD_FACTOR(existing_rtt)) {
- return true;
+ for (i = 0; i < NUMBER_SERVERS; i++) {
+ sorted_servers[i] = &dev->alt_servers[i];
}
- return false;
+ sort(sorted_servers, NUMBER_SERVERS, sizeof(struct dnbd3_device *),
+ &dnbd3_compare_servers, NULL);
+
+ return sorted_servers;
}
/**
- * dnbd3_adjust_connections - adjust the connections of the device
- * @dev: the device
+ * dnbd3_adjust_connections - create a connection plan and connect
+ * @dev: the dnbd3 device
*
- * 1. connect empty sockets if best alternative server is found
- * 2. replace slow socket with better server if available
- * 3. remove socket if one is slow
+ * 1. sort the alt server after the avg rtt
+ * 2. create a connection plan
+ * 3. connect the plan
*/
-static void dnbd3_adjust_connections(struct dnbd3_device *dev) {
- int i;
- int sock_alive = 0;
- uint64_t rtt;
- uint64_t best_rtt = RTT_UNREACHABLE;
- struct dnbd3_server *server, *existing_server;
+static int dnbd3_adjust_connections(struct dnbd3_device *dev) {
+ int i, j, fallback;
+ struct dnbd3_server *plan[NUMBER_CONNECTIONS];
+ struct dnbd3_server **servers = dnbd3_sort_server(dev);
- /* connect empty sockets */ //TODO check if panic worker connects at the same time, than we would have a problem
- sock_alive = 0;
- for (i = 0; i < NUMBER_CONNECTIONS; i++) {
- if (!dnbd3_is_sock_alive(dev->socks[i])) {
- server = dnbd3_find_best_alt_server(dev);
- if (server) {
- if (dnbd3_server_connect(dev, server) == 0) {
- sock_alive++;
+ if (servers && servers[0]->host.type != 0) {
+ mutex_lock(&dev->device_lock);
+ plan[0] = servers[0];
+ fallback = 0;
+ j = 1;
+ debug_dev(dev, "connection plan:");
+ debug_server(dev, plan[0], "server 0 with rtt %llu:",
+ plan[0]->avg_rtt);
+ for (i = 1; i < NUMBER_CONNECTIONS; i++) {
+ if (servers[j]->host.type != 0) {
+ if (RTT_FACTOR(plan[i - 1]->avg_rtt) >
+ servers[j]->avg_rtt) {
+ plan[i] = servers[j];
+ j++;
} else {
- warn_server(dev, server, "failed to connect");
+ plan[i] = plan[fallback];
+ fallback++;
}
+ } else {
+ plan[i] = plan[fallback];
+ fallback++;
}
- } else {
- sock_alive++;
+ debug_server(dev, plan[i], "server %d with rtt %llu:",
+ i, plan[i]->avg_rtt);
}
- }
+ kfree(servers);
- /* replace socket with better server */
- if (sock_alive == NUMBER_CONNECTIONS) {
for (i = 0; i < NUMBER_CONNECTIONS; i++) {
- if (dnbd3_is_sock_alive(dev->socks[i])) {
- server = dnbd3_find_best_alt_server(dev);
- existing_server = dev->socks[i].server;
- if (server && dnbd3_better_rtt(server, dev->socks[i].server)) {
+ if (plan[i] != dev->socks[i].server) {
+ if (dnbd3_is_sock_alive(dev->socks[i])) {
dnbd3_socket_disconnect(&dev->socks[i]);
-
- /* if it fails reconnect to existing */
- if (dnbd3_server_connect(dev, server) != 0) {
- warn_server(dev, server, "failed to connect");
- dnbd3_server_connect(dev, existing_server);
- }
- }
- }
- }
- }
-
- /* remove a socket if it is much slower than the others */
- if (sock_alive > 1) {
- for (i = 0; i < NUMBER_CONNECTIONS; i++) {
- if (dnbd3_is_sock_alive(dev->socks[i])) {
- rtt = dnbd3_avg_rtt(dev->socks[i].server);
- if (rtt <= best_rtt) {
- best_rtt = rtt;
}
+ dnbd3_socket_connect(&dev->socks[i], plan[i]);
}
}
- for (i = 0; i < NUMBER_CONNECTIONS; i++) {
- if (dnbd3_is_sock_alive(dev->socks[i])) {
- rtt = dnbd3_avg_rtt(dev->socks[i].server);
- if (rtt > RTT_THRESOULD_LIMIT(best_rtt)) {
- info_sock(&dev->socks[i], "removing connection with rtt %llu", rtt);
- dnbd3_socket_disconnect(&dev->socks[i]);
- sock_alive--;
- }
- }
+ mutex_unlock(&dev->device_lock);
+ return 0;
+ } else { /* there is nothing to connect */
+ if (servers) {
+ kfree(servers);
}
+ return -ENONET;
}
- info_dev(dev, "connected to %d/%d sockets", sock_alive, NUMBER_CONNECTIONS);
-}
-/**
- * dnbd3_panic_connect - connect to the first available server
- * @dev: the device
- */
-static int dnbd3_panic_connect(struct dnbd3_device *dev) {
- int result, i;
- result = dnbd3_server_connect(dev, &dev->initial_server);
- if (result) {
- for (i = 0; i < NUMBER_SERVERS; i++) {
- if (dev->alt_servers[i].host.type != 0) {
- result = dnbd3_server_connect(dev,
- &dev->alt_servers[i]);
- if (!result) {
- info_server(dev, &dev->alt_servers[i], "found server to connect to");
- break;
- }
- }
- }
- }
- return result;
}
/**
@@ -921,47 +818,27 @@ static int dnbd3_panic_connect(struct dnbd3_device *dev) {
static void dnbd3_panic_worker(struct work_struct *work)
{
struct dnbd3_device *dev;
- struct dnbd3_sock *panicked_sock = NULL;
- struct dnbd3_server *new_server, *panicked_server;
+ bool panic = false;
int i;
int sock_alive = 0;
dev = container_of(work, struct dnbd3_device, panic_worker);
for (i = 0; i < NUMBER_CONNECTIONS; i++) {
if (dev->socks[i].panic) {
- panicked_sock = &dev->socks[i];
+ panic = true;
+ dnbd3_set_rtt_unreachable(dev->socks[i].server);
+ dnbd3_socket_disconnect(&dev->socks[i]);
+
} else if (dnbd3_is_sock_alive(dev->socks[i])) {
sock_alive++;
}
}
- if (panicked_sock) {
- warn_sock(panicked_sock, "panicked, connections still alive %d",
- sock_alive);
- panicked_server = panicked_sock->server;
- new_server = dnbd3_find_best_alt_server(dev);
- dnbd3_socket_disconnect(panicked_sock);
- dnbd3_set_rtt_unreachable(panicked_server);
+ if (panic) {
+ warn_dev(dev, "panicked, connections still alive %d",
+ sock_alive);
- if (new_server != NULL && new_server != panicked_server) {
- info_server(dev, new_server, "found replacement");
- if (!dnbd3_server_connect(dev, new_server)) {
- sock_alive++;
- }
- } else if (sock_alive > 0) {
- info_dev(dev, "found no replacement server but still connected to %d servers",
- sock_alive);
- }
- }
-
- if (sock_alive == 0) {
- warn_dev(dev, "did not find a good server, trying to connect to any available server");
-
- if (dnbd3_panic_connect(dev)) {
- error_dev(dev, "could not connect to any server");
- } else {
- info_dev(dev, "found server to connect to");
- }
+ dnbd3_adjust_connections(dev);
}
}
@@ -985,7 +862,7 @@ static int dnbd3_meassure_rtt(struct dnbd3_device *dev,
.server = server
};
- result = dnbd3_socket_connect(&sock, server);
+ result = __dnbd3_socket_connect(&sock, server);
if (result) {
error_sock(&sock, "socket connect failed in rtt measurement");
goto error;
@@ -1044,6 +921,7 @@ static int dnbd3_meassure_rtt(struct dnbd3_device *dev,
error:
sock.server->rtts[dev->discovery_count % 4] = rtt;
+ sock.server->avg_rtt = dnbd3_avg_rtt(sock.server);
if (result <= 0) {
server->failures++;
}
@@ -1092,8 +970,9 @@ static void dnbd3_merge_new_server(struct dnbd3_device *dev,
if (new_server->failures == 1) { /* remove is requested */
info_server(dev, new_server,
"remove server is requested");
- dnbd3_server_disconnect(dev, existing_server);
+ // adjust connection will remove it later
existing_server->host.type = 0;
+ dnbd3_set_rtt_unreachable(existing_server);
}
// existing_server->failures = 0; // reset failure count
return;
@@ -1110,9 +989,10 @@ static void dnbd3_merge_new_server(struct dnbd3_device *dev,
info_server(dev, free_server, "got new alternative server");
free_server->failures = 0;
free_server->protocol_version = 0;
- dnbd3_set_rtt_unreachable(free_server);
+ dnbd3_set_rtt_unknown(free_server);
}
+
/**
* dnbd3_discovery_worker - handle discovery
* @work: the work used to get the dndb3_device
@@ -1168,11 +1048,11 @@ static void dnbd3_discovery_worker(struct work_struct *work)
/**
- * dnbd3_socket_connect - connect a socket to a server
+ * __dnbd3_socket_connect - internal connect a socket to a server
* @sock: the socket to connect
* @server: the server
*/
-static int dnbd3_socket_connect(struct dnbd3_sock *sock,
+static int __dnbd3_socket_connect(struct dnbd3_sock *sock,
struct dnbd3_server *server)
{
int result = 0;
@@ -1237,44 +1117,27 @@ error:
}
/**
- * dnbd3_server_connect - connect a server to a device
- * @dev: the device
+ * dnbd3_socket_connect - connect a socket to a server
+ * @sock: the socket
* @server: the server to connect
*
- * 1. connects the server to a free socket if available
+ * 1. connects the server to the socket
* 2. select the image
* 3. start receiver_worker and keepalive_worker
- * 4. if it is the first connection start timer, panic_worker and
- * keepalive_worker
- * 5. update the mq queues to the number of sockets alive
*/
-static int dnbd3_server_connect(struct dnbd3_device *dev,
+static int dnbd3_socket_connect(struct dnbd3_sock *sock,
struct dnbd3_server *server)
{
- int i;
- int sock_alive = 0;
int result = -EIO;
dnbd3_reply_t reply;
- struct dnbd3_sock *sock = NULL;
- for (i = 0; i < NUMBER_CONNECTIONS; i++) {
- if (!dnbd3_is_sock_alive(dev->socks[i])) {
- sock = &dev->socks[i];
- break;
- }
- }
- if (sock == NULL) {
- warn_server(dev, server, "could not connect to socket, to many connections");
- return -EIO;
- }
+ struct dnbd3_device *dev = sock->device;
sock->server = server;
-
debug_sock(sock, "socket connect");
-
mutex_init(&sock->tx_lock);
mutex_lock(&sock->tx_lock);
- result = dnbd3_socket_connect(sock, server);
+ result = __dnbd3_socket_connect(sock, server);
if (result) {
error_sock(sock, "connection to socket failed");
result = -EIO;
@@ -1294,6 +1157,7 @@ static int dnbd3_server_connect(struct dnbd3_device *dev,
result = dnbd3_send_request_cmd(sock, CMD_SELECT_IMAGE);
if (result <= 0) {
+
error_sock(sock, "connection to image %s failed", dev->imgname);
result = -EIO;
goto error;
@@ -1328,23 +1192,7 @@ static int dnbd3_server_connect(struct dnbd3_device *dev,
INIT_WORK(&sock->keepalive_worker, dnbd3_keepalive_worker);
- for (i = 0; i < NUMBER_CONNECTIONS; i++) {
- if (dev->socks[i].sock && dev->socks[i].server) {
- sock_alive++;
- }
- }
- /* if first socket to connect, start timer and workers */
- if (sock_alive == 1) {
- debug_sock(sock, "first connection to server, starting workers");
- INIT_WORK(&dev->discovery_worker, dnbd3_discovery_worker);
- INIT_WORK(&dev->panic_worker, dnbd3_panic_worker);
- timer_setup(&dev->timer, dnbd3_timer, 0);
- dev->timer.expires = jiffies + HZ;
- add_timer(&dev->timer);
- }
- blk_mq_update_nr_hw_queues(&dev->tag_set, sock_alive);
-
- /* request alternative servers receiver will handle this */
+ /* TODO not on every connect? request alternative servers receiver will handle this */
if (dnbd3_send_request_cmd(sock, CMD_GET_SERVERS) <= 0) {
error_sock(sock, "failed to get servers in discovery");
}
@@ -1352,6 +1200,7 @@ static int dnbd3_server_connect(struct dnbd3_device *dev,
return 0;
error:
server->failures++;
+ sock->panic = true;
if (sock->sock) {
kernel_sock_shutdown(sock->sock, SHUT_RDWR);
cancel_work_sync(&sock->receive_worker);
@@ -1362,41 +1211,13 @@ error:
return result;
}
-static void dnbd3_clear_req(struct request *req, void *data, bool reserved)
-{
- struct dnbd3_cmd *cmd = blk_mq_rq_to_pdu(req);
-
- cmd->status = BLK_STS_IOERR;
- blk_mq_complete_request(req);
-}
/**
- * dnbd3_socket_disconnect - disconnect a socket or server
+ * dnbd3_socket_disconnect - disconnect a socket
* @sock: the socket to disconnect
- *
- * 1. update nr of mq queues
- * 2. if last socket remove timer
- * 3. disconnect socket
*/
static int dnbd3_socket_disconnect(struct dnbd3_sock *sock)
{
- int i;
- struct dnbd3_device *dev = sock->device;
- int sock_alive = 0;
- for (i = 0; i < NUMBER_CONNECTIONS; i++) {
- if (dnbd3_is_sock_alive(dev->socks[i])) {
- sock_alive++;
- }
- }
- if (sock_alive <= 1) {
- info_sock(sock, "shutting down last socket and stopping timer");
- del_timer_sync(&dev->timer);
- /*
- * do not wait for discovery and panic worker as they may have
- * called this method
- */
-
- }
cancel_work_sync(&sock->keepalive_worker);
debug_sock(sock, "socket disconnect");
@@ -1425,50 +1246,18 @@ static int dnbd3_socket_disconnect(struct dnbd3_sock *sock)
sock_release(sock->sock);
sock->sock = NULL;
}
- debug_sock(sock, "update nr hw queues to %d", sock_alive -1);
-// blk_mq_quiesce_queue(dev->disk->queue);
-// debug_sock(sock, "busy iter");
-// blk_mq_tagset_busy_iter(&dev->tag_set, dnbd3_clear_req, NULL);
-// debug_sock(sock, "update");
- //TODO can we just stop the failed queue?
- blk_mq_update_nr_hw_queues(&dev->tag_set, sock_alive - 1);
-// debug_sock(sock, "unquisco");
-// blk_mq_unquiesce_queue(dev->disk->queue);
-// debug_sock(sock, "donee");
sock->server = NULL;
sock->panic = false;
return 0;
}
-/**
- * dnbd3_server_disconnect - disconnect a server from a socket
- * @dev: the device
- * @server: the server to disconnect
- */
-static int dnbd3_server_disconnect(struct dnbd3_device *dev,
- struct dnbd3_server *server)
-{
- int i;
- struct dnbd3_sock *sock = NULL;
- for (i = 0; i < NUMBER_CONNECTIONS; i++) {
- if (dev->socks[i].server == server) {
- sock = &dev->socks[i];
- }
- }
- if (!sock || !sock->sock) {
- warn_dev(dev, "could not find socket to disconnect");
- return -EIO;
- }
- return dnbd3_socket_disconnect(sock);
-}
-
/**
* dnbd3_net_connect - connect device
* @dev: the device to connect
*
- * dnbd3_device.alt_servers[0] must set
+ * dnbd3_device.alt_servers[0] must be set
*/
int dnbd3_net_connect(struct dnbd3_device *dev)
{
@@ -1479,14 +1268,27 @@ int dnbd3_net_connect(struct dnbd3_device *dev)
return -ENONET;
}
- // alt_server[0] is the initial server
- result = dnbd3_server_connect(dev, &dev->alt_servers[0]);
+ result = dnbd3_adjust_connections(dev);
if (result) {
error_dev(dev, "failed to connect to initial server");
- result = -ENOENT;
- dev->imgname = NULL;
- dev->socks[0].server = NULL;
+ return -ENOENT;
}
+
+ debug_dev(dev, "connected, starting workers");
+ INIT_WORK(&dev->discovery_worker, dnbd3_discovery_worker);
+ INIT_WORK(&dev->panic_worker, dnbd3_panic_worker);
+ timer_setup(&dev->timer, dnbd3_timer, 0);
+ dev->timer.expires = jiffies + HZ;
+ add_timer(&dev->timer);
+
+ // alt_server[0] is the initial server
+// result = dnbd3_server_connect(dev, &dev->alt_servers[0]);
+// if (result) {
+// error_dev(dev, "failed to connect to initial server");
+// result = -ENOENT;
+// dev->imgname = NULL;
+// dev->socks[0].server = NULL;
+// }
return result;
}
diff --git a/src/kernel/sysfs.c b/src/kernel/sysfs.c
index 1588dfb..490ffbd 100644
--- a/src/kernel/sysfs.c
+++ b/src/kernel/sysfs.c
@@ -46,18 +46,18 @@ ssize_t show_connected_servers(char *buf, struct dnbd3_device *dev)
if (dev->socks[i].sock) {
if (dev->socks[i].server->host.type == HOST_IP4) {
ret = MIN(snprintf(buf, size, "%pI4,%d,%llu,%d\n",
- dev->socks[i].server->host.addr,
- (int)ntohs(dev->socks[i].server->host.port),
- (unsigned long long)((dev->socks[i].server->rtts[0] + dev->socks[i].server->rtts[1] + dev->socks[i].server->rtts[2] + dev->socks[i].server->rtts[3]) / 4),
- (int)dev->socks[i].server->failures)
+ dev->socks[i].server->host.addr,
+ (int)ntohs(dev->socks[i].server->host.port),
+ dev->socks[i].server->avg_rtt,
+ (int)dev->socks[i].server->failures)
, size);
} else {
ret = MIN(snprintf(buf, size, "%pI6,%d,%llu,%d\n",
- dev->socks[i].server->host.addr,
- (int)ntohs(dev->socks[i].server->host.port),
- (unsigned long long)((dev->socks[i].server->rtts[0] + dev->socks[i].server->rtts[1] + dev->socks[i].server->rtts[2] + dev->socks[i].server->rtts[3]) / 4),
- (int)dev->socks[i].server->failures)
+ dev->socks[i].server->host.addr,
+ (int)ntohs(dev->socks[i].server->host.port),
+ dev->socks[i].server->avg_rtt,
+ (int)dev->socks[i].server->failures)
, size);
}
@@ -81,9 +81,10 @@ ssize_t show_connected_servers(char *buf, struct dnbd3_device *dev)
ssize_t show_alt_server_num(char *buf, struct dnbd3_device *dev)
{
int i, num = 0;
- for (i = 0; i < NUMBER_SERVERS; ++i)
- {
- if (dev->alt_servers[i].host.type) ++num;
+ for (i = 0; i < NUMBER_SERVERS; i++) {
+ if (dev->alt_servers[i].host.type) {
+ num++;
+ }
}
return MIN(snprintf(buf, PAGE_SIZE, "%d\n", num), PAGE_SIZE);
}
@@ -91,28 +92,27 @@ ssize_t show_alt_server_num(char *buf, struct dnbd3_device *dev)
ssize_t show_alt_servers(char *buf, struct dnbd3_device *dev)
{
int i, size = PAGE_SIZE, ret;
- for (i = 0; i < NUMBER_SERVERS; ++i)
- {
- if (dev->alt_servers[i].host.type == HOST_IP4)
+ for (i = 0; i < NUMBER_SERVERS; ++i) {
+ if (dev->alt_servers[i].host.type == HOST_IP4) {
ret = MIN(snprintf(buf, size, "%pI4,%d,%llu,%d\n",
dev->alt_servers[i].host.addr,
(int)ntohs(dev->alt_servers[i].host.port),
- (unsigned long long)((dev->alt_servers[i].rtts[0] + dev->alt_servers[i].rtts[1] + dev->alt_servers[i].rtts[2] + dev->alt_servers[i].rtts[3]) / 4),
+ dev->alt_servers[i].avg_rtt,
(int)dev->alt_servers[i].failures)
, size);
- else if (dev->alt_servers[i].host.type == HOST_IP6)
+ } else if (dev->alt_servers[i].host.type == HOST_IP6) {
ret = MIN(snprintf(buf, size, "%pI6,%d,%llu,%d\n",
dev->alt_servers[i].host.addr,
(int)ntohs(dev->alt_servers[i].host.port),
- (unsigned long long)((dev->alt_servers[i].rtts[0] + dev->alt_servers[i].rtts[1] + dev->alt_servers[i].rtts[2] + dev->alt_servers[i].rtts[3]) / 4),
+ dev->alt_servers[i].avg_rtt,
(int)dev->alt_servers[i].failures)
, size);
- else
+ } else {
continue;
+ }
size -= ret;
buf += ret;
- if (size <= 0)
- {
+ if (size <= 0) {
size = 0;
break;
}