From e0f69d83d5c5c039b133b60b5a7130dedeeaca42 Mon Sep 17 00:00:00 2001 From: Vladimir Sementsov-Ogievskiy Date: Sat, 3 Jul 2021 00:16:35 +0300 Subject: iotest 151: add test-case that shows active mirror dead-lock There is a dead-lock in active mirror: when we have parallel intersecting requests (note that non intersecting requests may be considered intersecting after aligning to mirror granularity), it may happen that request A waits request B in mirror_wait_on_conflicts() and request B waits for A. Look at the test for details. Test now dead-locks, that's why it's disabled. Next commit will fix mirror and enable the test. Signed-off-by: Vladimir Sementsov-Ogievskiy Message-Id: <20210702211636.228981-3-vsementsov@virtuozzo.com> Signed-off-by: Kevin Wolf --- tests/qemu-iotests/151 | 62 ++++++++++++++++++++++++++++++++++++++++++++-- tests/qemu-iotests/151.out | 4 +-- 2 files changed, 62 insertions(+), 4 deletions(-) (limited to 'tests') diff --git a/tests/qemu-iotests/151 b/tests/qemu-iotests/151 index 182f6b5321..ab46c5e8ba 100755 --- a/tests/qemu-iotests/151 +++ b/tests/qemu-iotests/151 @@ -38,8 +38,9 @@ class TestActiveMirror(iotests.QMPTestCase): 'if': 'none', 'node-name': 'source-node', 'driver': iotests.imgfmt, - 'file': {'driver': 'file', - 'filename': source_img}} + 'file': {'driver': 'blkdebug', + 'image': {'driver': 'file', + 'filename': source_img}}} blk_target = {'node-name': 'target-node', 'driver': iotests.imgfmt, @@ -141,6 +142,63 @@ class TestActiveMirror(iotests.QMPTestCase): self.potential_writes_in_flight = False + def testIntersectingActiveIO(self): + # FIXME: test-case is dead-locking. To reproduce dead-lock just drop + # this return statement + return + + # Fill the source image + result = self.vm.hmp_qemu_io('source', 'write -P 1 0 2M') + + # Start the block job (very slowly) + result = self.vm.qmp('blockdev-mirror', + job_id='mirror', + filter_node_name='mirror-node', + device='source-node', + target='target-node', + sync='full', + copy_mode='write-blocking', + speed=1) + + self.vm.hmp_qemu_io('source', 'break write_aio A') + self.vm.hmp_qemu_io('source', 'aio_write 0 1M') # 1 + self.vm.hmp_qemu_io('source', 'wait_break A') + self.vm.hmp_qemu_io('source', 'aio_write 0 2M') # 2 + self.vm.hmp_qemu_io('source', 'aio_write 0 2M') # 3 + + # Now 2 and 3 are in mirror_wait_on_conflicts, waiting for 1 + + self.vm.hmp_qemu_io('source', 'break write_aio B') + self.vm.hmp_qemu_io('source', 'aio_write 1M 2M') # 4 + self.vm.hmp_qemu_io('source', 'wait_break B') + + # 4 doesn't wait for 2 and 3, because they didn't yet set + # in_flight_bitmap. So, nothing prevents 4 to go except for our + # break-point B. + + self.vm.hmp_qemu_io('source', 'resume A') + + # Now we resumed 1, so 2 and 3 goes to the next iteration of while loop + # in mirror_wait_on_conflicts(). They don't exit, as bitmap is dirty + # due to request 4. And they start to wait: 2 wait for 3, 3 wait for 2 + # - DEAD LOCK. + # Note that it's important that we add request 4 at last: requests are + # appended to the list, so we are sure that 4 is last in the list, so 2 + # and 3 now waits for each other, not for 4. + + self.vm.hmp_qemu_io('source', 'resume B') + + # Resuming 4 doesn't help, 2 and 3 already dead-locked + # To check the dead-lock run: + # gdb -p $(pidof qemu-system-x86_64) -ex 'set $job=(MirrorBlockJob *)jobs.lh_first' -ex 'p *$job->ops_in_flight.tqh_first' -ex 'p *$job->ops_in_flight.tqh_first->next.tqe_next' + # You'll see two MirrorOp objects waiting on each other + + result = self.vm.qmp('block-job-set-speed', device='mirror', speed=0) + self.assert_qmp(result, 'return', {}) + self.complete_and_wait(drive='mirror') + + self.potential_writes_in_flight = False + if __name__ == '__main__': iotests.main(supported_fmts=['qcow2', 'raw'], diff --git a/tests/qemu-iotests/151.out b/tests/qemu-iotests/151.out index 8d7e996700..89968f35d7 100644 --- a/tests/qemu-iotests/151.out +++ b/tests/qemu-iotests/151.out @@ -1,5 +1,5 @@ -... +.... ---------------------------------------------------------------------- -Ran 3 tests +Ran 4 tests OK -- cgit v1.2.3-55-g7522 From d44dae1a7cf782ec9235746ebb0e6c1a20dd7288 Mon Sep 17 00:00:00 2001 From: Vladimir Sementsov-Ogievskiy Date: Sat, 3 Jul 2021 00:16:36 +0300 Subject: block/mirror: fix active mirror dead-lock in mirror_wait_on_conflicts It's possible that requests start to wait each other in mirror_wait_on_conflicts(). To avoid it let's use same technique as in block/io.c in bdrv_wait_serialising_requests_locked() / bdrv_find_conflicting_request(): don't wait on intersecting request if it is already waiting for some other request. For details of the dead-lock look at testIntersectingActiveIO() test-case which we actually fixing now. Fixes: d06107ade0ce74dc39739bac80de84b51ec18546 Signed-off-by: Vladimir Sementsov-Ogievskiy Message-Id: <20210702211636.228981-4-vsementsov@virtuozzo.com> Signed-off-by: Kevin Wolf --- block/mirror.c | 12 ++++++++++++ tests/qemu-iotests/151 | 18 +++++------------- 2 files changed, 17 insertions(+), 13 deletions(-) (limited to 'tests') diff --git a/block/mirror.c b/block/mirror.c index ad6aac2f95..98fc66eabf 100644 --- a/block/mirror.c +++ b/block/mirror.c @@ -107,6 +107,7 @@ struct MirrorOp { bool is_in_flight; CoQueue waiting_requests; Coroutine *co; + MirrorOp *waiting_for_op; QTAILQ_ENTRY(MirrorOp) next; }; @@ -159,7 +160,18 @@ static void coroutine_fn mirror_wait_on_conflicts(MirrorOp *self, if (ranges_overlap(self_start_chunk, self_nb_chunks, op_start_chunk, op_nb_chunks)) { + /* + * If the operation is already (indirectly) waiting for us, or + * will wait for us as soon as it wakes up, then just go on + * (instead of producing a deadlock in the former case). + */ + if (op->waiting_for_op) { + continue; + } + + self->waiting_for_op = op; qemu_co_queue_wait(&op->waiting_requests, NULL); + self->waiting_for_op = NULL; break; } } diff --git a/tests/qemu-iotests/151 b/tests/qemu-iotests/151 index ab46c5e8ba..93d14193d0 100755 --- a/tests/qemu-iotests/151 +++ b/tests/qemu-iotests/151 @@ -143,10 +143,6 @@ class TestActiveMirror(iotests.QMPTestCase): self.potential_writes_in_flight = False def testIntersectingActiveIO(self): - # FIXME: test-case is dead-locking. To reproduce dead-lock just drop - # this return statement - return - # Fill the source image result = self.vm.hmp_qemu_io('source', 'write -P 1 0 2M') @@ -180,18 +176,14 @@ class TestActiveMirror(iotests.QMPTestCase): # Now we resumed 1, so 2 and 3 goes to the next iteration of while loop # in mirror_wait_on_conflicts(). They don't exit, as bitmap is dirty - # due to request 4. And they start to wait: 2 wait for 3, 3 wait for 2 - # - DEAD LOCK. - # Note that it's important that we add request 4 at last: requests are - # appended to the list, so we are sure that 4 is last in the list, so 2 - # and 3 now waits for each other, not for 4. + # due to request 4. + # In the past at that point 2 and 3 would wait for each other producing + # a dead-lock. Now this is fixed and they will wait for request 4. self.vm.hmp_qemu_io('source', 'resume B') - # Resuming 4 doesn't help, 2 and 3 already dead-locked - # To check the dead-lock run: - # gdb -p $(pidof qemu-system-x86_64) -ex 'set $job=(MirrorBlockJob *)jobs.lh_first' -ex 'p *$job->ops_in_flight.tqh_first' -ex 'p *$job->ops_in_flight.tqh_first->next.tqe_next' - # You'll see two MirrorOp objects waiting on each other + # After resuming 4, one of 2 and 3 goes first and set in_flight_bitmap, + # so the other will wait for it. result = self.vm.qmp('block-job-set-speed', device='mirror', speed=0) self.assert_qmp(result, 'return', {}) -- cgit v1.2.3-55-g7522 From d21471696b07f30cb00453709d055a25c1afde85 Mon Sep 17 00:00:00 2001 From: Max Reitz Date: Thu, 24 Jun 2021 10:38:25 +0200 Subject: iotests/307: Test iothread conflict for exports Passing fixed-iothread=true should make iothread conflicts fatal, whereas fixed-iothread=false should not. Combine the second case with an error condition that is checked after the iothread is handled, to verify that qemu does not crash if there is such an error after changing the iothread failed. Signed-off-by: Max Reitz Message-Id: <20210624083825.29224-3-mreitz@redhat.com> Reviewed-by: Vladimir Sementsov-Ogievskiy Tested-by: Vladimir Sementsov-Ogievskiy Signed-off-by: Kevin Wolf --- tests/qemu-iotests/307 | 15 +++++++++++++++ tests/qemu-iotests/307.out | 8 ++++++++ 2 files changed, 23 insertions(+) (limited to 'tests') diff --git a/tests/qemu-iotests/307 b/tests/qemu-iotests/307 index c7685347bc..b429b5aa50 100755 --- a/tests/qemu-iotests/307 +++ b/tests/qemu-iotests/307 @@ -41,9 +41,11 @@ with iotests.FilePath('image') as img, \ iotests.log('=== Launch VM ===') vm.add_object('iothread,id=iothread0') + vm.add_object('iothread,id=iothread1') vm.add_blockdev(f'file,filename={img},node-name=file') vm.add_blockdev(f'{iotests.imgfmt},file=file,node-name=fmt') vm.add_blockdev('raw,file=file,node-name=ro,read-only=on') + vm.add_blockdev('null-co,node-name=null') vm.add_device(f'id=scsi0,driver=virtio-scsi,iothread=iothread0') vm.launch() @@ -74,6 +76,19 @@ with iotests.FilePath('image') as img, \ vm.qmp_log('query-block-exports') iotests.qemu_nbd_list_log('-k', socket) + iotests.log('\n=== Add export with conflicting iothread ===') + + vm.qmp_log('device_add', id='sdb', driver='scsi-hd', drive='null') + + # Should fail because of fixed-iothread + vm.qmp_log('block-export-add', id='export1', type='nbd', node_name='null', + iothread='iothread1', fixed_iothread=True, writable=True) + + # Should ignore the iothread conflict, but then fail because of the + # permission conflict (and not crash) + vm.qmp_log('block-export-add', id='export1', type='nbd', node_name='null', + iothread='iothread1', fixed_iothread=False, writable=True) + iotests.log('\n=== Add a writable export ===') # This fails because share-rw=off diff --git a/tests/qemu-iotests/307.out b/tests/qemu-iotests/307.out index 4b0c7e155a..ec8d2be0e0 100644 --- a/tests/qemu-iotests/307.out +++ b/tests/qemu-iotests/307.out @@ -51,6 +51,14 @@ exports available: 1 base:allocation +=== Add export with conflicting iothread === +{"execute": "device_add", "arguments": {"drive": "null", "driver": "scsi-hd", "id": "sdb"}} +{"return": {}} +{"execute": "block-export-add", "arguments": {"fixed-iothread": true, "id": "export1", "iothread": "iothread1", "node-name": "null", "type": "nbd", "writable": true}} +{"error": {"class": "GenericError", "desc": "Cannot change iothread of active block backend"}} +{"execute": "block-export-add", "arguments": {"fixed-iothread": false, "id": "export1", "iothread": "iothread1", "node-name": "null", "type": "nbd", "writable": true}} +{"error": {"class": "GenericError", "desc": "Permission conflict on node 'null': permissions 'write' are both required by an unnamed block device (uses node 'null' as 'root' child) and unshared by block device 'sdb' (uses node 'null' as 'root' child)."}} + === Add a writable export === {"execute": "block-export-add", "arguments": {"description": "This is the writable second export", "id": "export1", "name": "export1", "node-name": "fmt", "type": "nbd", "writable": true, "writethrough": true}} {"error": {"class": "GenericError", "desc": "Permission conflict on node 'fmt': permissions 'write' are both required by an unnamed block device (uses node 'fmt' as 'root' child) and unshared by block device 'sda' (uses node 'fmt' as 'root' child)."}} -- cgit v1.2.3-55-g7522