From 3c2d5183f9fa4eac3d17d841e26da65a0181ae7b Mon Sep 17 00:00:00 2001 From: Stefan Hajnoczi Date: Tue, 29 Aug 2017 13:27:43 +0100 Subject: nbd-client: avoid read_reply_co entry if send failed The following segfault is encountered if the NBD server closes the UNIX domain socket immediately after negotiation: Program terminated with signal SIGSEGV, Segmentation fault. #0 aio_co_schedule (ctx=0x0, co=0xd3c0ff2ef0) at util/async.c:441 441 QSLIST_INSERT_HEAD_ATOMIC(&ctx->scheduled_coroutines, (gdb) bt #0 0x000000d3c01a50f8 in aio_co_schedule (ctx=0x0, co=0xd3c0ff2ef0) at util/async.c:441 #1 0x000000d3c012fa90 in nbd_coroutine_end (bs=bs@entry=0xd3c0fec650, request=) at block/nbd-client.c:207 #2 0x000000d3c012fb58 in nbd_client_co_preadv (bs=0xd3c0fec650, offset=0, bytes=, qiov=0x7ffc10a91b20, flags=0) at block/nbd-client.c:237 #3 0x000000d3c0128e63 in bdrv_driver_preadv (bs=bs@entry=0xd3c0fec650, offset=offset@entry=0, bytes=bytes@entry=512, qiov=qiov@entry=0x7ffc10a91b20, flags=0) at block/io.c:836 #4 0x000000d3c012c3e0 in bdrv_aligned_preadv (child=child@entry=0xd3c0ff51d0, req=req@entry=0x7f31885d6e90, offset=offset@entry=0, bytes=bytes@entry=512, align=align@entry=1, qiov=qiov@entry=0x7ffc10a91b20, f +lags=0) at block/io.c:1086 #5 0x000000d3c012c6b8 in bdrv_co_preadv (child=0xd3c0ff51d0, offset=offset@entry=0, bytes=bytes@entry=512, qiov=qiov@entry=0x7ffc10a91b20, flags=flags@entry=0) at block/io.c:1182 #6 0x000000d3c011cc17 in blk_co_preadv (blk=0xd3c0ff4f80, offset=0, bytes=512, qiov=0x7ffc10a91b20, flags=0) at block/block-backend.c:1032 #7 0x000000d3c011ccec in blk_read_entry (opaque=0x7ffc10a91b40) at block/block-backend.c:1079 #8 0x000000d3c01bbb96 in coroutine_trampoline (i0=, i1=) at util/coroutine-ucontext.c:79 #9 0x00007f3196cb8600 in __start_context () at /lib64/libc.so.6 The problem is that nbd_client_init() uses nbd_client_attach_aio_context() -> aio_co_schedule(new_context, client->read_reply_co). Execution of read_reply_co is deferred to a BH which doesn't run until later. In the mean time blk_co_preadv() can be called and nbd_coroutine_end() calls aio_wake() on read_reply_co. At this point in time read_reply_co's ctx isn't set because it has never been entered yet. This patch simplifies the nbd_co_send_request() -> nbd_co_receive_reply() -> nbd_coroutine_end() lifecycle to just nbd_co_send_request() -> nbd_co_receive_reply(). The request is "ended" if an error occurs at any point. Callers no longer have to invoke nbd_coroutine_end(). This cleanup also eliminates the segfault because we don't call aio_co_schedule() to wake up s->read_reply_co if sending the request failed. It is only necessary to wake up s->read_reply_co if a reply was received. Note this only happens with UNIX domain sockets on Linux. It doesn't seem possible to reproduce this with TCP sockets. Suggested-by: Paolo Bonzini Signed-off-by: Stefan Hajnoczi Message-Id: <20170829122745.14309-2-stefanha@redhat.com> Signed-off-by: Eric Blake --- block/nbd-client.c | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) (limited to 'block') diff --git a/block/nbd-client.c b/block/nbd-client.c index 25bcaa2346..ea728fffc8 100644 --- a/block/nbd-client.c +++ b/block/nbd-client.c @@ -144,12 +144,12 @@ static int nbd_co_send_request(BlockDriverState *bs, request->handle = INDEX_TO_HANDLE(s, i); if (s->quit) { - qemu_co_mutex_unlock(&s->send_mutex); - return -EIO; + rc = -EIO; + goto err; } if (!s->ioc) { - qemu_co_mutex_unlock(&s->send_mutex); - return -EPIPE; + rc = -EPIPE; + goto err; } if (qiov) { @@ -166,8 +166,13 @@ static int nbd_co_send_request(BlockDriverState *bs, } else { rc = nbd_send_request(s->ioc, request); } + +err: if (rc < 0) { s->quit = true; + s->requests[i].coroutine = NULL; + s->in_flight--; + qemu_co_queue_next(&s->free_sema); } qemu_co_mutex_unlock(&s->send_mutex); return rc; @@ -201,13 +206,6 @@ static void nbd_co_receive_reply(NBDClientSession *s, /* Tell the read handler to read another header. */ s->reply.handle = 0; } -} - -static void nbd_coroutine_end(BlockDriverState *bs, - NBDRequest *request) -{ - NBDClientSession *s = nbd_get_client_session(bs); - int i = HANDLE_TO_INDEX(s, request->handle); s->requests[i].coroutine = NULL; @@ -243,7 +241,6 @@ int nbd_client_co_preadv(BlockDriverState *bs, uint64_t offset, } else { nbd_co_receive_reply(client, &request, &reply, qiov); } - nbd_coroutine_end(bs, &request); return -reply.error; } @@ -272,7 +269,6 @@ int nbd_client_co_pwritev(BlockDriverState *bs, uint64_t offset, } else { nbd_co_receive_reply(client, &request, &reply, NULL); } - nbd_coroutine_end(bs, &request); return -reply.error; } @@ -306,7 +302,6 @@ int nbd_client_co_pwrite_zeroes(BlockDriverState *bs, int64_t offset, } else { nbd_co_receive_reply(client, &request, &reply, NULL); } - nbd_coroutine_end(bs, &request); return -reply.error; } @@ -330,7 +325,6 @@ int nbd_client_co_flush(BlockDriverState *bs) } else { nbd_co_receive_reply(client, &request, &reply, NULL); } - nbd_coroutine_end(bs, &request); return -reply.error; } @@ -355,7 +349,6 @@ int nbd_client_co_pdiscard(BlockDriverState *bs, int64_t offset, int bytes) } else { nbd_co_receive_reply(client, &request, &reply, NULL); } - nbd_coroutine_end(bs, &request); return -reply.error; } -- cgit v1.2.3-55-g7522 From 6faa077772db366e5d2198ec0a14a50d0ccde1c6 Mon Sep 17 00:00:00 2001 From: Vladimir Sementsov-Ogievskiy Date: Fri, 4 Aug 2017 18:14:28 +0300 Subject: block/nbd-client: get rid of ssize_t Use int variable for nbd_co_send_request return value (as nbd_co_send_request returns int). Signed-off-by: Vladimir Sementsov-Ogievskiy Message-Id: <20170804151440.320927-6-vsementsov@virtuozzo.com> Signed-off-by: Eric Blake --- block/nbd-client.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'block') diff --git a/block/nbd-client.c b/block/nbd-client.c index ea728fffc8..1e393cf26f 100644 --- a/block/nbd-client.c +++ b/block/nbd-client.c @@ -230,7 +230,7 @@ int nbd_client_co_preadv(BlockDriverState *bs, uint64_t offset, .len = bytes, }; NBDReply reply; - ssize_t ret; + int ret; assert(bytes <= NBD_MAX_BUFFER_SIZE); assert(!flags); @@ -254,7 +254,7 @@ int nbd_client_co_pwritev(BlockDriverState *bs, uint64_t offset, .len = bytes, }; NBDReply reply; - ssize_t ret; + int ret; if (flags & BDRV_REQ_FUA) { assert(client->info.flags & NBD_FLAG_SEND_FUA); @@ -275,7 +275,7 @@ int nbd_client_co_pwritev(BlockDriverState *bs, uint64_t offset, int nbd_client_co_pwrite_zeroes(BlockDriverState *bs, int64_t offset, int bytes, BdrvRequestFlags flags) { - ssize_t ret; + int ret; NBDClientSession *client = nbd_get_client_session(bs); NBDRequest request = { .type = NBD_CMD_WRITE_ZEROES, @@ -310,7 +310,7 @@ int nbd_client_co_flush(BlockDriverState *bs) NBDClientSession *client = nbd_get_client_session(bs); NBDRequest request = { .type = NBD_CMD_FLUSH }; NBDReply reply; - ssize_t ret; + int ret; if (!(client->info.flags & NBD_FLAG_SEND_FLUSH)) { return 0; @@ -337,7 +337,7 @@ int nbd_client_co_pdiscard(BlockDriverState *bs, int64_t offset, int bytes) .len = bytes, }; NBDReply reply; - ssize_t ret; + int ret; if (!(client->info.flags & NBD_FLAG_SEND_TRIM)) { return 0; -- cgit v1.2.3-55-g7522 From 07b1b99c78e4b9479922fbe2d05a87ef810e6b7e Mon Sep 17 00:00:00 2001 From: Vladimir Sementsov-Ogievskiy Date: Fri, 4 Aug 2017 18:14:31 +0300 Subject: block/nbd-client: rename nbd_recv_coroutines_enter_all Rename nbd_recv_coroutines_enter_all to nbd_recv_coroutines_wake_all, as it most probably just adds all recv coroutines into co_queue_wakeup, rather than directly enter them. Signed-off-by: Vladimir Sementsov-Ogievskiy Message-Id: <20170804151440.320927-9-vsementsov@virtuozzo.com> [eblake: tweak commit message] Signed-off-by: Eric Blake --- block/nbd-client.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'block') diff --git a/block/nbd-client.c b/block/nbd-client.c index 1e393cf26f..322b725ff9 100644 --- a/block/nbd-client.c +++ b/block/nbd-client.c @@ -34,7 +34,7 @@ #define HANDLE_TO_INDEX(bs, handle) ((handle) ^ ((uint64_t)(intptr_t)bs)) #define INDEX_TO_HANDLE(bs, index) ((index) ^ ((uint64_t)(intptr_t)bs)) -static void nbd_recv_coroutines_enter_all(NBDClientSession *s) +static void nbd_recv_coroutines_wake_all(NBDClientSession *s) { int i; @@ -112,7 +112,7 @@ static coroutine_fn void nbd_read_reply_entry(void *opaque) } s->quit = true; - nbd_recv_coroutines_enter_all(s); + nbd_recv_coroutines_wake_all(s); s->read_reply_co = NULL; } -- cgit v1.2.3-55-g7522 From f35dff7e13b84d3fffe1103c2c69afd81df5e4f5 Mon Sep 17 00:00:00 2001 From: Vladimir Sementsov-Ogievskiy Date: Tue, 29 Aug 2017 16:48:31 -0500 Subject: block/nbd-client: refactor request send/receive Add nbd_co_request, to remove code duplications in nbd_client_co_{pwrite,pread,...} functions. Also this is needed for further refactoring. Signed-off-by: Vladimir Sementsov-Ogievskiy Message-Id: <20170804151440.320927-8-vsementsov@virtuozzo.com> [eblake: make nbd_co_request a wrapper, rather than merging two existing functions] Signed-off-by: Eric Blake --- block/nbd-client.c | 73 +++++++++++++++++++----------------------------------- 1 file changed, 26 insertions(+), 47 deletions(-) (limited to 'block') diff --git a/block/nbd-client.c b/block/nbd-client.c index 322b725ff9..f0dbea24d3 100644 --- a/block/nbd-client.c +++ b/block/nbd-client.c @@ -220,28 +220,40 @@ static void nbd_co_receive_reply(NBDClientSession *s, qemu_co_mutex_unlock(&s->send_mutex); } +static int nbd_co_request(BlockDriverState *bs, + NBDRequest *request, + QEMUIOVector *qiov) +{ + NBDClientSession *client = nbd_get_client_session(bs); + NBDReply reply; + int ret; + + assert(!qiov || request->type == NBD_CMD_WRITE || + request->type == NBD_CMD_READ); + ret = nbd_co_send_request(bs, request, + request->type == NBD_CMD_WRITE ? qiov : NULL); + if (ret < 0) { + reply.error = -ret; + } else { + nbd_co_receive_reply(client, request, &reply, + request->type == NBD_CMD_READ ? qiov : NULL); + } + return -reply.error; +} + int nbd_client_co_preadv(BlockDriverState *bs, uint64_t offset, uint64_t bytes, QEMUIOVector *qiov, int flags) { - NBDClientSession *client = nbd_get_client_session(bs); NBDRequest request = { .type = NBD_CMD_READ, .from = offset, .len = bytes, }; - NBDReply reply; - int ret; assert(bytes <= NBD_MAX_BUFFER_SIZE); assert(!flags); - ret = nbd_co_send_request(bs, &request, NULL); - if (ret < 0) { - reply.error = -ret; - } else { - nbd_co_receive_reply(client, &request, &reply, qiov); - } - return -reply.error; + return nbd_co_request(bs, &request, qiov); } int nbd_client_co_pwritev(BlockDriverState *bs, uint64_t offset, @@ -253,8 +265,6 @@ int nbd_client_co_pwritev(BlockDriverState *bs, uint64_t offset, .from = offset, .len = bytes, }; - NBDReply reply; - int ret; if (flags & BDRV_REQ_FUA) { assert(client->info.flags & NBD_FLAG_SEND_FUA); @@ -263,26 +273,18 @@ int nbd_client_co_pwritev(BlockDriverState *bs, uint64_t offset, assert(bytes <= NBD_MAX_BUFFER_SIZE); - ret = nbd_co_send_request(bs, &request, qiov); - if (ret < 0) { - reply.error = -ret; - } else { - nbd_co_receive_reply(client, &request, &reply, NULL); - } - return -reply.error; + return nbd_co_request(bs, &request, qiov); } int nbd_client_co_pwrite_zeroes(BlockDriverState *bs, int64_t offset, int bytes, BdrvRequestFlags flags) { - int ret; NBDClientSession *client = nbd_get_client_session(bs); NBDRequest request = { .type = NBD_CMD_WRITE_ZEROES, .from = offset, .len = bytes, }; - NBDReply reply; if (!(client->info.flags & NBD_FLAG_SEND_WRITE_ZEROES)) { return -ENOTSUP; @@ -296,21 +298,13 @@ int nbd_client_co_pwrite_zeroes(BlockDriverState *bs, int64_t offset, request.flags |= NBD_CMD_FLAG_NO_HOLE; } - ret = nbd_co_send_request(bs, &request, NULL); - if (ret < 0) { - reply.error = -ret; - } else { - nbd_co_receive_reply(client, &request, &reply, NULL); - } - return -reply.error; + return nbd_co_request(bs, &request, NULL); } int nbd_client_co_flush(BlockDriverState *bs) { NBDClientSession *client = nbd_get_client_session(bs); NBDRequest request = { .type = NBD_CMD_FLUSH }; - NBDReply reply; - int ret; if (!(client->info.flags & NBD_FLAG_SEND_FLUSH)) { return 0; @@ -319,13 +313,7 @@ int nbd_client_co_flush(BlockDriverState *bs) request.from = 0; request.len = 0; - ret = nbd_co_send_request(bs, &request, NULL); - if (ret < 0) { - reply.error = -ret; - } else { - nbd_co_receive_reply(client, &request, &reply, NULL); - } - return -reply.error; + return nbd_co_request(bs, &request, NULL); } int nbd_client_co_pdiscard(BlockDriverState *bs, int64_t offset, int bytes) @@ -336,21 +324,12 @@ int nbd_client_co_pdiscard(BlockDriverState *bs, int64_t offset, int bytes) .from = offset, .len = bytes, }; - NBDReply reply; - int ret; if (!(client->info.flags & NBD_FLAG_SEND_TRIM)) { return 0; } - ret = nbd_co_send_request(bs, &request, NULL); - if (ret < 0) { - reply.error = -ret; - } else { - nbd_co_receive_reply(client, &request, &reply, NULL); - } - return -reply.error; - + return nbd_co_request(bs, &request, NULL); } void nbd_client_detach_aio_context(BlockDriverState *bs) -- cgit v1.2.3-55-g7522