diff options
author | Frederic Robra | 2019-07-05 17:30:02 +0200 |
---|---|---|
committer | Frederic Robra | 2019-07-05 17:30:02 +0200 |
commit | 0de54378aff9348e45c9ccdd4310887a753acfab (patch) | |
tree | d4dd3caa7824ab6d928dc5893782db92e440f520 | |
parent | fixed some errors with multi queue (diff) | |
download | dnbd3-ng-0de54378aff9348e45c9ccdd4310887a753acfab.tar.gz dnbd3-ng-0de54378aff9348e45c9ccdd4310887a753acfab.tar.xz dnbd3-ng-0de54378aff9348e45c9ccdd4310887a753acfab.zip |
added first support for connection with more than one server
-rw-r--r-- | src/kernel/block.c | 351 | ||||
-rw-r--r-- | src/kernel/block.h | 2 | ||||
-rw-r--r-- | src/kernel/core.c | 20 | ||||
-rw-r--r-- | src/kernel/dnbd3.h | 40 | ||||
-rw-r--r-- | src/kernel/mq.c | 75 | ||||
-rw-r--r-- | src/kernel/sysfs.c | 58 |
6 files changed, 245 insertions, 301 deletions
diff --git a/src/kernel/block.c b/src/kernel/block.c index ade02f0..4295418 100644 --- a/src/kernel/block.c +++ b/src/kernel/block.c @@ -22,241 +22,211 @@ void dnbd3_blk_fail_all_requests(dnbd3_device_t *dev) } - -int dnbd3_net_connect(dnbd3_device_t *dev) +int dnbd3_socket_connect(dnbd3_device_t *dev, dnbd3_sock *sock) { + int result = -EIO; struct request *req1 = NULL; struct timeval timeout; - - printk(KERN_DEBUG "dnbd3: net connect device %i\n", dev->minor); - - if (dev->disconnecting) { - printk(KERN_DEBUG "dnbd3: device still disconnecting\n"); - while (dev->disconnecting) + struct dnbd3_server_t *server = sock->server; + dnbd3_request_t dnbd3_request; + dnbd3_reply_t dnbd3_reply; + struct msghdr msg; + struct kvec iov[2]; + uint16_t rid; + uint64_t reported_size; + char *name; + int mlen; + serialized_buffer_t payload_buffer; + + printk(KERN_DEBUG "dnbd3: socket connect device %i\n", dev->minor); + + mutex_init(&sock->lock); + mutex_lock(&sock->lock); + if (sock->pending) { + printk(KERN_DEBUG "dnbd3: socket still in request\n"); + while (sock->pending) schedule(); } - if (dev->pending) { - printk(KERN_DEBUG "dnbd3: device still in request\n"); - while (dev->disconnecting) - schedule(); + if (server->host.port == 0 || server->host.type == 0) { + printk(KERN_ERR "dnbd3: host or port not set\n"); + goto error; + } + if (sock->sock) { + printk(KERN_WARNING "dnbd3: socket already connected\n"); + goto error; } timeout.tv_sec = SOCKET_TIMEOUT_CLIENT_DATA; timeout.tv_usec = 0; - // do some checks before connecting - req1 = kmalloc(sizeof(*req1), GFP_ATOMIC ); if (!req1) { printk(KERN_ERR "dnbd3: kmalloc failed\n"); goto error; } - if (dev->cur_server.host.port == 0 || dev->cur_server.host.type == 0 || dev->imgname == NULL ) { - printk(KERN_ERR "dnbd3: host, port or image name not set\n"); - goto error; - } - if (dev->sock) { - printk(KERN_WARNING "dnbd3: already connected\n"); - goto error; - } - if (dev->cur_server.host.type != HOST_IP4 && dev->cur_server.host.type != HOST_IP6) { - printk(KERN_ERR "dnbd3: unknown address type %d\n", (int)dev->cur_server.host.type); + init_msghdr(msg); + + if (dnbd3_sock_create(server->host.type, SOCK_STREAM, IPPROTO_TCP, &sock->sock) < 0) { + printk(KERN_ERR "dnbd3: could not create socket\n"); goto error; } - if (dev->better_sock == NULL ) { - // no established connection yet from discovery thread, start new one - dnbd3_request_t dnbd3_request; - dnbd3_reply_t dnbd3_reply; - struct msghdr msg; - struct kvec iov[2]; - uint16_t rid; - char *name; - int mlen; - init_msghdr(msg); - - if (dnbd3_sock_create(dev->cur_server.host.type, SOCK_STREAM, IPPROTO_TCP, &dev->sock) < 0) { - printk(KERN_ERR "dnbd3: could not create socket\n"); + kernel_setsockopt(sock->sock, SOL_SOCKET, SO_SNDTIMEO, (char *)&timeout, sizeof(timeout)); + kernel_setsockopt(sock->sock, SOL_SOCKET, SO_RCVTIMEO, (char *)&timeout, sizeof(timeout)); + sock->sock->sk->sk_allocation = GFP_NOIO; + if (server->host.type == HOST_IP4) { + struct sockaddr_in sin; + memset(&sin, 0, sizeof(sin)); + sin.sin_family = AF_INET; + memcpy(&(sin.sin_addr), server->host.addr, 4); + sin.sin_port = server->host.port; + if (kernel_connect(sock->sock, (struct sockaddr *)&sin, sizeof(sin), 0) != 0) { + printk(KERN_ERR "dnbd3: connection to host failed (ipv4)\n"); goto error; } - - kernel_setsockopt(dev->sock, SOL_SOCKET, SO_SNDTIMEO, (char *)&timeout, sizeof(timeout)); - kernel_setsockopt(dev->sock, SOL_SOCKET, SO_RCVTIMEO, (char *)&timeout, sizeof(timeout)); - dev->sock->sk->sk_allocation = GFP_NOIO; - if (dev->cur_server.host.type == HOST_IP4) { - struct sockaddr_in sin; - memset(&sin, 0, sizeof(sin)); - sin.sin_family = AF_INET; - memcpy(&(sin.sin_addr), dev->cur_server.host.addr, 4); - sin.sin_port = dev->cur_server.host.port; - if (kernel_connect(dev->sock, (struct sockaddr *)&sin, sizeof(sin), 0) != 0) { - printk(KERN_ERR "dnbd3: connection to host failed (ipv4)\n"); - goto error; - } - } else { - struct sockaddr_in6 sin; - memset(&sin, 0, sizeof(sin)); - sin.sin6_family = AF_INET6; - memcpy(&(sin.sin6_addr), dev->cur_server.host.addr, 16); - sin.sin6_port = dev->cur_server.host.port; - if (kernel_connect(dev->sock, (struct sockaddr *)&sin, sizeof(sin), 0) != 0){ - printk(KERN_ERR "dnbd3: connection to host failed (ipv6)\n"); - goto error; - } - } - // Request filesize - dnbd3_request.magic = dnbd3_packet_magic; - dnbd3_request.cmd = CMD_SELECT_IMAGE; - iov[0].iov_base = &dnbd3_request; - iov[0].iov_len = sizeof(dnbd3_request); - serializer_reset_write(&dev->payload_buffer); - serializer_put_uint16(&dev->payload_buffer, PROTOCOL_VERSION); - serializer_put_string(&dev->payload_buffer, dev->imgname); - serializer_put_uint16(&dev->payload_buffer, dev->rid); - serializer_put_uint8(&dev->payload_buffer, 0); // is_server = false - iov[1].iov_base = &dev->payload_buffer; - dnbd3_request.size = iov[1].iov_len = serializer_get_written_length(&dev->payload_buffer); - fixup_request(dnbd3_request); - mlen = sizeof(dnbd3_request) + iov[1].iov_len; - if (kernel_sendmsg(dev->sock, &msg, iov, 2, mlen) != mlen) { - printk(KERN_ERR "dnbd3: could not send CMD_SIZE_REQUEST\n"); - goto error; - } - // receive reply header - iov[0].iov_base = &dnbd3_reply; - iov[0].iov_len = sizeof(dnbd3_reply); - if (kernel_recvmsg(dev->sock, &msg, iov, 1, sizeof(dnbd3_reply), msg.msg_flags) != sizeof(dnbd3_reply)) { - printk(KERN_ERR "dnbd3: received corrupted reply header after CMD_SIZE_REQUEST\n"); - goto error; - } - - // check reply header - fixup_reply(dnbd3_reply); - if (dnbd3_reply.cmd != CMD_SELECT_IMAGE || - dnbd3_reply.size < 3 || - dnbd3_reply.size > MAX_PAYLOAD || - dnbd3_reply.magic != dnbd3_packet_magic) { - printk(KERN_ERR "dnbd3: received invalid reply to CMD_SIZE_REQUEST image does not exist on server\n"); + } else { + struct sockaddr_in6 sin; + memset(&sin, 0, sizeof(sin)); + sin.sin6_family = AF_INET6; + memcpy(&(sin.sin6_addr), server->host.addr, 16); + sin.sin6_port = server->host.port; + if (kernel_connect(sock->sock, (struct sockaddr *)&sin, sizeof(sin), 0) != 0){ + printk(KERN_ERR "dnbd3: connection to host failed (ipv6)\n"); goto error; } + } + // Request filesize + dnbd3_request.magic = dnbd3_packet_magic; + dnbd3_request.cmd = CMD_SELECT_IMAGE; + iov[0].iov_base = &dnbd3_request; + iov[0].iov_len = sizeof(dnbd3_request); + serializer_reset_write(&payload_buffer); + serializer_put_uint16(&payload_buffer, PROTOCOL_VERSION); + serializer_put_string(&payload_buffer, dev->imgname); + serializer_put_uint16(&payload_buffer, dev->rid); + serializer_put_uint8(&payload_buffer, 0); // is_server = false + iov[1].iov_base = &payload_buffer; + dnbd3_request.size = iov[1].iov_len = serializer_get_written_length(&payload_buffer); + fixup_request(dnbd3_request); + mlen = sizeof(dnbd3_request) + iov[1].iov_len; + if (kernel_sendmsg(sock->sock, &msg, iov, 2, mlen) != mlen) { + printk(KERN_ERR "dnbd3: could not send CMD_SIZE_REQUEST\n"); + goto error; + } + // receive reply header + iov[0].iov_base = &dnbd3_reply; + iov[0].iov_len = sizeof(dnbd3_reply); + if (kernel_recvmsg(sock->sock, &msg, iov, 1, sizeof(dnbd3_reply), msg.msg_flags) != sizeof(dnbd3_reply)) { + printk(KERN_ERR "dnbd3: received corrupted reply header after CMD_SIZE_REQUEST\n"); + goto error; + } - // receive reply payload - iov[0].iov_base = &dev->payload_buffer; - iov[0].iov_len = dnbd3_reply.size; - if (kernel_recvmsg(dev->sock, &msg, iov, 1, dnbd3_reply.size, msg.msg_flags) != dnbd3_reply.size) { - printk(KERN_ERR "dnbd3: could not read CMD_SELECT_IMAGE payload on handshake\n"); - goto error; - } + // check reply header + fixup_reply(dnbd3_reply); + if (dnbd3_reply.cmd != CMD_SELECT_IMAGE || + dnbd3_reply.size < 3 || + dnbd3_reply.size > MAX_PAYLOAD || + dnbd3_reply.magic != dnbd3_packet_magic) { + printk(KERN_ERR "dnbd3: received invalid reply to CMD_SIZE_REQUEST image does not exist on server\n"); + goto error; + } - // handle/check reply payload - serializer_reset_read(&dev->payload_buffer, dnbd3_reply.size); - dev->cur_server.protocol_version = serializer_get_uint16(&dev->payload_buffer); - if (dev->cur_server.protocol_version < MIN_SUPPORTED_SERVER) { - printk(KERN_ERR "dnbd3: server version is lower than min supported version\n"); - goto error; - } + // receive reply payload + iov[0].iov_base = &payload_buffer; + iov[0].iov_len = dnbd3_reply.size; + if (kernel_recvmsg(sock->sock, &msg, iov, 1, dnbd3_reply.size, msg.msg_flags) != dnbd3_reply.size) { + printk(KERN_ERR "dnbd3: could not read CMD_SELECT_IMAGE payload on handshake\n"); + goto error; + } - name = serializer_get_string(&dev->payload_buffer); - if (dev->rid != 0 && strcmp(name, dev->imgname) != 0) { - printk(KERN_ERR "dnbd3: server offers image '%s', requested '%s'\n", name, dev->imgname); - goto error; - } + // handle/check reply payload + serializer_reset_read(&payload_buffer, dnbd3_reply.size); + server->protocol_version = serializer_get_uint16(&payload_buffer); + if (server->protocol_version < MIN_SUPPORTED_SERVER) { + printk(KERN_ERR "dnbd3: server version is lower than min supported version\n"); + goto error; + } - if (strlen(dev->imgname) < strlen(name)) { - dev->imgname = krealloc(dev->imgname, strlen(name) + 1, GFP_ATOMIC ); - if (dev->imgname == NULL ) { - printk(KERN_ERR "dnbd3: reallocating buffer for new image name failed\n"); - goto error; - } - } - strcpy(dev->imgname, name); - rid = serializer_get_uint16(&dev->payload_buffer); - if (dev->rid != 0 && dev->rid != rid) { - printk(KERN_ERR "dnbd3: server provides rid %d, requested was %d\n", (int)rid, (int)dev->rid); - goto error; - } + name = serializer_get_string(&payload_buffer); + rid = serializer_get_uint16(&payload_buffer); + if (dev->rid != rid && strcmp(name, dev->imgname) != 0) { + printk(KERN_ERR "dnbd3: server offers image '%s', requested '%s'\n", name, dev->imgname); + goto error; + } - dev->rid = rid; - dev->reported_size = serializer_get_uint64(&dev->payload_buffer); - if (dev->reported_size < 4096) { + reported_size = serializer_get_uint64(&payload_buffer); + if (dev->reported_size == NULL) { + if (reported_size < 4096) { printk(KERN_ERR "dnbd3: reported size by server is < 4096\n"); goto error; } - - // store image information + dev->reported_size = reported_size; set_capacity(dev->disk, dev->reported_size >> 9); /* 512 Byte blocks */ - printk(KERN_DEBUG "dnbd3: connected to image %s, filesize %llu\n", dev->imgname, dev->reported_size); - dev->update_available = 0; - } else { // Switching server, connection is already established and size request was executed - - printk(KERN_DEBUG "dnbd3: changing server\n"); - dev->sock = dev->better_sock; - dev->better_sock = NULL; - kernel_setsockopt(dev->sock, SOL_SOCKET, SO_SNDTIMEO, (char *)&timeout, sizeof(timeout)); - kernel_setsockopt(dev->sock, SOL_SOCKET, SO_RCVTIMEO, (char *)&timeout, sizeof(timeout)); + } else if (dev->reported_size != reported_size) { + printk(KERN_ERR "dnbd3: reported size by server is %llu but should be %llu\n", reported_size, dev->reported_size); } - dev->panic = 0; - dev->panic_count = 0; + printk(KERN_DEBUG "dnbd3: connected to image %s, filesize %llu\n", dev->imgname, dev->reported_size); + + mutex_unlock(&sock->lock); - refcount_inc(&dev->config_refs); - mutex_init(&dev->socket_lock); - // Enqueue request to request_queue_send for a fresh list of alt servers - //TODO refresh alt server list +// TODO add heartbeat + // timer_setup(&dev->hb_timer, dnbd3_net_heartbeat, 0); + // dev->hb_timer.expires = jiffies + HZ; + // add_timer(&dev->hb_timer); - // add heartbeat timer - //TODO needed anymore= -// dev->heartbeat_count = 0; -// timer_setup(&dev->hb_timer, dnbd3_net_heartbeat, 0); -// dev->hb_timer.expires = jiffies + HZ; -// add_timer(&dev->hb_timer); return 0; error: - if (dev->sock) { - sock_release(dev->sock); - dev->sock = NULL; + if (sock->sock) { + sock_release(sock->sock); + sock->sock = NULL; } - dev->cur_server.host.type = 0; - dev->cur_server.host.port = 0; if (req1) { kfree(req1); } - return -1; + mutex_unlock(&sock->lock); + return result; } - -int dnbd3_net_disconnect(dnbd3_device_t *dev) +int dnbd3_socket_disconnect(dnbd3_device_t *dev, dnbd3_sock *sock) { - printk(KERN_DEBUG "dnbd3: net disconnect device %i\n", dev->minor); - if (dev->disconnecting) { - return 0; - } - - dev->disconnecting = 1; + printk(KERN_DEBUG "dnbd3: socket disconnect device %i\n", dev->minor); + mutex_lock(&sock->lock); // clear heartbeat timer // del_timer(&dev->hb_timer); - dev->discover = 0; - if (dev->sock) { - kernel_sock_shutdown(dev->sock, SHUT_RDWR); + if (sock->sock) { + kernel_sock_shutdown(sock->sock, SHUT_RDWR); } // clear socket - if (dev->sock) { - sock_release(dev->sock); - dev->sock = NULL; + if (sock->sock) { + sock_release(sock->sock); + sock->sock = NULL; } - dev->cur_server.host.type = 0; - dev->cur_server.host.port = 0; - - dev->disconnecting = 0; + mutex_unlock(&sock->lock); + mutex_destroy(&sock->lock); return 0; } +int dnbd3_net_disconnect(struct dnbd3_device_t *dev) { + int i; + int result; + for (i = 0; i < NUMBER_CONNECTIONS; i++) { + if (dev->socks[i].sock) { + if (dnbd3_socket_disconnect(dev, &dev->socks[i])) { + result = -EIO; + } + } + } + return result; +} + + static void printHost(struct dnbd3_host_t *host, char *msg) { if (host->type == HOST_IP4) { @@ -269,7 +239,7 @@ static void printHost(struct dnbd3_host_t *host, char *msg) static void printServerList(struct dnbd3_device_t *dev) { int i; - printHost(&dev->cur_server.host, "current server is"); + printHost(&dev->initial_server.host, "initial server is"); for (i = 0; i < NUMBER_SERVERS; i++) { if (dev->alt_servers[i].host.addr[0] != 0) { printHost(&dev->alt_servers[i].host, "alternative server is"); @@ -283,13 +253,14 @@ static int dnbd3_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd dnbd3_device_t *dev = bdev->bd_disk->private_data; char *imgname = NULL; dnbd3_ioctl_t *msg = NULL; + int i; printk(KERN_DEBUG "dnbd3: ioctl device %i, cmd %i, arg %lu\n", dev->minor, cmd, arg); //unsigned long irqflags; - while (dev->disconnecting) { - // do nothing - } +// while (dev->disconnecting) { +// // do nothing +// } if (arg != 0) { msg = kmalloc(sizeof(*msg), GFP_KERNEL); @@ -328,12 +299,12 @@ static int dnbd3_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd } else if (msg == NULL) { result = -EINVAL; } else { - if (sizeof(msg->host) != sizeof(dev->cur_server.host)) { + if (sizeof(msg->host) != sizeof(dev->initial_server.host)) { printk(KERN_INFO "dnbd3: odd size bug#1 triggered in ioctl\n"); } - memcpy(&dev->cur_server.host, &msg->host, sizeof(msg->host)); - dev->cur_server.failures = 0; - memcpy(&dev->initial_server, &dev->cur_server, sizeof(dev->initial_server)); + memcpy(&dev->initial_server.host, &msg->host, sizeof(msg->host)); + dev->initial_server.failures = 0; +// memcpy(&dev->initial_server, &dev->cur_server, sizeof(dev->initial_server)); dev->imgname = imgname; dev->rid = msg->rid; dev->use_server_provided_alts = msg->use_server_provided_alts; @@ -348,22 +319,26 @@ static int dnbd3_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd // blk_queue->backing_dev_info.ra_pages = (msg->read_ahead_kb * 1024) / PAGE_SIZE; //#endif - printServerList(dev); - if (dnbd3_net_connect(dev) == 0) { + dev->socks[0].server = &dev->initial_server; + if (dnbd3_socket_connect(dev, &dev->socks[0]) == 0) { + printServerList(dev); result = 0; imgname = NULL; // Prevent kfree at the end } else { + printk(KERN_ERR "dnbd3: failed to connect to initial server\n"); result = -ENOENT; dev->imgname = NULL; + dev->socks[0].server = NULL; } + } break; case IOCTL_CLOSE: printk(KERN_DEBUG "dnbd3: ioctl close\n"); dnbd3_blk_fail_all_requests(dev); - result = dnbd3_net_disconnect(dev); + dnbd3_net_disconnect(dev); dnbd3_blk_fail_all_requests(dev); set_capacity(dev->disk, 0); if (dev->imgname) { diff --git a/src/kernel/block.h b/src/kernel/block.h index 07bf950..49055d5 100644 --- a/src/kernel/block.h +++ b/src/kernel/block.h @@ -16,7 +16,7 @@ h.msg_flags = MSG_WAITALL | MSG_NOSIGNAL; \ } while (0) -int dnbd3_net_connect(dnbd3_device_t *dev); +//int dnbd3_net_connect(dnbd3_device_t *dev); int dnbd3_net_disconnect(dnbd3_device_t *dev); diff --git a/src/kernel/core.c b/src/kernel/core.c index e92bd9a..db1f6de 100644 --- a/src/kernel/core.c +++ b/src/kernel/core.c @@ -120,16 +120,12 @@ static int dnbd3_add_device(dnbd3_device_t *dev, int minor) blk_queue_max_hw_sectors(disk->queue, 65536); disk->queue->limits.max_sectors = 256; - mutex_init(&dev->config_lock); - refcount_set(&dev->config_refs, 0); - refcount_set(&dev->refs, 1); INIT_LIST_HEAD(&dev->list); disk->major = major; disk->first_minor = minor; disk->fops = &dnbd3_fops; disk->private_data = dev; sprintf(disk->disk_name, "dnbd%i", minor); -// sprintf(disk->disk_name, "dnbd3%i", minor); printk(KERN_DEBUG "dnbd3: add disk device %s\n", disk->disk_name); add_disk(disk); dnbd3_sysfs_init(dev); @@ -207,9 +203,7 @@ static void dnbd3_dev_remove(struct dnbd3_device_t *dnbd3) del_gendisk(disk); blk_cleanup_queue(q); blk_mq_free_tag_set(&dnbd3->tag_set); - if (dnbd3->sock) { - dnbd3_net_disconnect(dnbd3); - } + dnbd3_net_disconnect(dnbd3); disk->private_data = NULL; put_disk(disk); } @@ -217,11 +211,10 @@ static void dnbd3_dev_remove(struct dnbd3_device_t *dnbd3) static void dnbd3_put(struct dnbd3_device_t *dnbd3) { - if (refcount_dec_and_mutex_lock(&dnbd3->refs, &dnbd3_index_mutex)) { - idr_remove(&dnbd3_index_idr, dnbd3->minor); - mutex_unlock(&dnbd3_index_mutex); - dnbd3_dev_remove(dnbd3); - } + mutex_lock(&dnbd3_index_mutex); + idr_remove(&dnbd3_index_idr, dnbd3->minor); + mutex_unlock(&dnbd3_index_mutex); + dnbd3_dev_remove(dnbd3); } @@ -239,9 +232,6 @@ static void __exit dnbd3_exit(void) dnbd3 = list_first_entry(&del_list, struct dnbd3_device_t, list); dnbd3_sysfs_exit(dnbd3); list_del_init(&dnbd3->list); - if (refcount_read(&dnbd3->refs) != 1) { - printk(KERN_ERR "dnbd3: possibly leaking a device\n"); - } dnbd3_put(dnbd3); } diff --git a/src/kernel/dnbd3.h b/src/kernel/dnbd3.h index 0a4ef67..7ad6b2f 100644 --- a/src/kernel/dnbd3.h +++ b/src/kernel/dnbd3.h @@ -31,9 +31,9 @@ #include "serialize.h" +#define NUMBER_CONNECTIONS 4 -typedef struct -{ +typedef struct dnbd3_server_t { dnbd3_host_t host; uint64_t rtts[4]; // Last four round trip time measurements in microsecond uint16_t protocol_version; // dnbd3 protocol version of this server @@ -41,13 +41,16 @@ typedef struct } dnbd3_server_t; +typedef struct dnbd3_sock { + struct socket *sock; + struct mutex lock; + struct request *pending; + struct dnbd3_server_t *server; +} dnbd3_sock; + typedef struct dnbd3_device_t { int minor; struct blk_mq_tag_set tag_set; -// struct request_queue queue; - struct mutex config_lock; - refcount_t config_refs; - refcount_t refs; struct list_head list; // block @@ -57,23 +60,26 @@ typedef struct dnbd3_device_t { struct kobject kobj; // network + dnbd3_sock socks[NUMBER_CONNECTIONS]; char *imgname; - struct socket *sock; - struct mutex socket_lock; - struct request *pending; - dnbd3_server_t cur_server, initial_server; - uint64_t cur_rtt; - serialized_buffer_t payload_buffer; +// struct socket *sock; +// struct mutex socket_lock; +// struct request *pending; + dnbd3_server_t initial_server; +// dnbd3_server_t cur_server, initial_server; +// uint64_t cur_rtt; +// serialized_buffer_t payload_buffer; dnbd3_server_t alt_servers[NUMBER_SERVERS]; // array of alt servers int new_servers_num; // number of new alt servers that are waiting to be copied to above array dnbd3_server_entry_t new_servers[NUMBER_SERVERS]; // pending new alt servers - uint8_t discover, panic, disconnecting, update_available, panic_count; +// uint8_t discover, panic, disconnecting, update_available, panic_count; + uint8_t update_available; uint8_t use_server_provided_alts; uint16_t rid; uint32_t heartbeat_count; uint64_t reported_size; // server switch - struct socket *better_sock; +// struct socket *better_sock; // process @@ -100,11 +106,5 @@ typedef struct dnbd3_cmd { uint32_t cmd_cookie; } dnbd3_cmd; -typedef struct dnbd3_sock { - struct socket *sock; - struct mutex lock; - struct request *pending; - struct dnbd3_server_t *server; -} dnbd3_sock; #endif /* DNBD_H_ */ diff --git a/src/kernel/mq.c b/src/kernel/mq.c index 3b0bde5..98e6e14 100644 --- a/src/kernel/mq.c +++ b/src/kernel/mq.c @@ -11,27 +11,11 @@ #include "block.h" #define DNBD3_CMD_REQUEUED 1 -// -//static void dnbd3_config_put(struct dnbd3_device_t *dev) -//{ -// if (refcount_dec_and_mutex_lock(&dev->config_refs, -// &dev->config_lock)) { -// -// dev->tag_set.timeout = 0; -// dev->disk->queue->limits.discard_granularity = 0; -// dev->disk->queue->limits.discard_alignment = 0; -// blk_queue_max_discard_sectors(dev->disk->queue, UINT_MAX); -// blk_queue_flag_clear(QUEUE_FLAG_DISCARD, dev->disk->queue); -// -// mutex_unlock(&dev->config_lock); -// nbd_put(dev); -// module_put(THIS_MODULE); -// } -//} + #define dnbd3_priv_to_cmd(req) ((req)->cmd_flags >> REQ_FLAG_BITS) -static int dnbd3_send_cmd(struct dnbd3_device_t *dev, struct dnbd3_cmd *cmd, int index) +static int dnbd3_send_cmd(struct dnbd3_device_t *dev, struct dnbd3_sock *sock, struct dnbd3_cmd *cmd, int index) { struct request *req = blk_mq_rq_from_pdu(cmd); dnbd3_request_t dnbd3_request; @@ -44,7 +28,7 @@ static int dnbd3_send_cmd(struct dnbd3_device_t *dev, struct dnbd3_cmd *cmd, int sigset_t blocked, oldset; void *kaddr; int result; - dev->pending = req; + sock->pending = req; init_msghdr(msg); dnbd3_request.magic = dnbd3_packet_magic; @@ -79,7 +63,7 @@ static int dnbd3_send_cmd(struct dnbd3_device_t *dev, struct dnbd3_cmd *cmd, int fixup_request(dnbd3_request); iov.iov_base = &dnbd3_request; iov.iov_len = sizeof(dnbd3_request); - if (kernel_sendmsg(dev->sock, &msg, &iov, 1, sizeof(dnbd3_request)) != sizeof(dnbd3_request)) { + if (kernel_sendmsg(sock->sock, &msg, &iov, 1, sizeof(dnbd3_request)) != sizeof(dnbd3_request)) { printk(KERN_ERR "dnbd3: connection to server lost\n"); result = -EIO; goto error; @@ -88,7 +72,7 @@ static int dnbd3_send_cmd(struct dnbd3_device_t *dev, struct dnbd3_cmd *cmd, int // receive net reply iov.iov_base = &dnbd3_reply; iov.iov_len = sizeof(dnbd3_reply); - result = kernel_recvmsg(dev->sock, &msg, &iov, 1, sizeof(dnbd3_reply), msg.msg_flags); + result = kernel_recvmsg(sock->sock, &msg, &iov, 1, sizeof(dnbd3_reply), msg.msg_flags); if (!result) { printk(KERN_ERR "dnbd3: connection to server lost\n"); result = -EIO; @@ -119,7 +103,7 @@ static int dnbd3_send_cmd(struct dnbd3_device_t *dev, struct dnbd3_cmd *cmd, int kaddr = kmap(bvec->bv_page) + bvec->bv_offset; iov.iov_base = kaddr; iov.iov_len = bvec->bv_len; - if (kernel_recvmsg(dev->sock, &msg, &iov, 1, bvec->bv_len, msg.msg_flags) != bvec->bv_len) { + if (kernel_recvmsg(sock->sock, &msg, &iov, 1, bvec->bv_len, msg.msg_flags) != bvec->bv_len) { kunmap(bvec->bv_page); sigprocmask(SIG_SETMASK, &oldset, NULL ); printk(KERN_ERR "dnbd3: could not receive form net to block layer\n"); @@ -129,10 +113,8 @@ static int dnbd3_send_cmd(struct dnbd3_device_t *dev, struct dnbd3_cmd *cmd, int sigprocmask(SIG_SETMASK, &oldset, NULL ); } -// -// __blk_end_request_all(req, 0); blk_mq_end_request(req, 0); - dev->pending = NULL; + sock->pending = NULL; error: return result; } @@ -150,57 +132,54 @@ static int dnbd3_handle_cmd(struct dnbd3_cmd *cmd, int index) { struct request *req = blk_mq_rq_from_pdu(cmd); struct dnbd3_device_t *dev = cmd->dnbd3; + struct dnbd3_sock *sock; int ret = -1; printk(KERN_DEBUG "dnbd3: handle request at position %lu and size %d, device %i\n", blk_rq_pos(req), blk_rq_bytes(req), dev->minor); - if (!refcount_inc_not_zero(&dev->config_refs)) { - dev_err_ratelimited(disk_to_dev(dev->disk), "socks array is empty\n"); - blk_mq_start_request(req); - return -EINVAL; - } +// if (index >= 1) { // TODO use next server with good rtt for this request +// printk(KERN_INFO "dnbd3: index is %d", index); +// dev_err_ratelimited(disk_to_dev(dev->disk), "attempted send on invalid socket\n"); +// blk_mq_start_request(req); +// return -EINVAL; +// } - if (index >= 1) { // TODO use next server with good rtt for this request - printk(KERN_INFO "dnbd3: index is %d", index); + sock = &dev->socks[index]; + if (!sock->sock) { + printk(KERN_INFO "dnbd3: index is %d but no socket was found\n", index); dev_err_ratelimited(disk_to_dev(dev->disk), "attempted send on invalid socket\n"); -// nbd_config_put(nbd); -// TODO what to do here? blk_mq_start_request(req); return -EINVAL; } + + cmd->status = BLK_STS_OK; again: - mutex_lock(&dev->socket_lock); - if (!dev->sock) { - mutex_unlock(&dev->socket_lock); - printk(KERN_DEBUG "dnbd3: not connected, try to reconnect\n"); - if (!dnbd3_net_connect(dev)) { - printk(KERN_ERR "dnbd3: failed to reconnect\n"); - blk_mq_start_request(req); - return -EIO; - } - goto again; + mutex_lock(&sock->lock); + if (unlikely(!sock->sock)) { + mutex_unlock(&sock->lock); + printk(KERN_DEBUG "dnbd3: not connected\n"); + return -EIO; } blk_mq_start_request(req); - if (unlikely(dev->pending && dev->pending != req)) { + if (unlikely(sock->pending && sock->pending != req)) { dnbd3_requeue_cmd(cmd); ret = 0; goto out; } - ret = dnbd3_send_cmd(dev, cmd, index); + ret = dnbd3_send_cmd(dev, sock, cmd, index); if (ret == -EAGAIN) { dev_err_ratelimited(disk_to_dev(dev->disk), "request send failed, requeueing\n"); dnbd3_requeue_cmd(cmd); ret = 0; } out: - mutex_unlock(&dev->socket_lock); -// nbd_config_put(nbd); + mutex_unlock(&sock->lock); return ret; } diff --git a/src/kernel/sysfs.c b/src/kernel/sysfs.c index 4406072..118cbca 100644 --- a/src/kernel/sysfs.c +++ b/src/kernel/sysfs.c @@ -27,20 +27,20 @@ #define MIN(a,b) ((a) < (b) ? (a) : (b)) #endif -ssize_t show_cur_server_addr(char *buf, dnbd3_device_t *dev) -{ - if (dev->cur_server.host.type == HOST_IP4) - return MIN(snprintf(buf, PAGE_SIZE, "%pI4,%d\n", dev->cur_server.host.addr, (int)ntohs(dev->cur_server.host.port)), PAGE_SIZE); - else if (dev->cur_server.host.type == HOST_IP6) - return MIN(snprintf(buf, PAGE_SIZE, "%pI6,%d\n", dev->cur_server.host.addr, (int)ntohs(dev->cur_server.host.port)), PAGE_SIZE); - *buf = '\0'; - return 0; -} - -ssize_t show_cur_server_rtt(char *buf, dnbd3_device_t *dev) -{ - return MIN(snprintf(buf, PAGE_SIZE, "%llu\n", (unsigned long long)dev->cur_rtt), PAGE_SIZE); -} +//ssize_t show_cur_server_addr(char *buf, dnbd3_device_t *dev) +//{ +// if (dev->cur_server.host.type == HOST_IP4) +// return MIN(snprintf(buf, PAGE_SIZE, "%pI4,%d\n", dev->cur_server.host.addr, (int)ntohs(dev->cur_server.host.port)), PAGE_SIZE); +// else if (dev->cur_server.host.type == HOST_IP6) +// return MIN(snprintf(buf, PAGE_SIZE, "%pI6,%d\n", dev->cur_server.host.addr, (int)ntohs(dev->cur_server.host.port)), PAGE_SIZE); +// *buf = '\0'; +// return 0; +//} +// +//ssize_t show_cur_server_rtt(char *buf, dnbd3_device_t *dev) +//{ +// return MIN(snprintf(buf, PAGE_SIZE, "%llu\n", (unsigned long long)dev->cur_rtt), PAGE_SIZE); +//} ssize_t show_alt_server_num(char *buf, dnbd3_device_t *dev) { @@ -100,19 +100,19 @@ ssize_t show_update_available(char *buf, dnbd3_device_t *dev) return MIN(snprintf(buf, PAGE_SIZE, "%d\n", dev->update_available), PAGE_SIZE); } -device_attr_t cur_server_addr = -{ - .attr = {.name = "cur_server_addr", .mode = 0444 }, - .show = show_cur_server_addr, - .store = NULL, -}; - -device_attr_t cur_server_rtt = -{ - .attr = {.name = "cur_server_rtt", .mode = 0444 }, - .show = show_cur_server_rtt, - .store = NULL, -}; +//device_attr_t cur_server_addr = +//{ +// .attr = {.name = "cur_server_addr", .mode = 0444 }, +// .show = show_cur_server_addr, +// .store = NULL, +//}; +// +//device_attr_t cur_server_rtt = +//{ +// .attr = {.name = "cur_server_rtt", .mode = 0444 }, +// .show = show_cur_server_rtt, +// .store = NULL, +//}; device_attr_t alt_server_num = { @@ -158,8 +158,8 @@ ssize_t device_show(struct kobject *kobj, struct attribute *attr, char *buf) struct attribute *device_attrs[] = { - &cur_server_addr.attr, - &cur_server_rtt.attr, +// &cur_server_addr.attr, +// &cur_server_rtt.attr, &alt_server_num.attr, &alt_servers.attr, &image_name.attr, |