summaryrefslogtreecommitdiffstats
path: root/tests
diff options
context:
space:
mode:
Diffstat (limited to 'tests')
-rwxr-xr-xtests/qemu-iotests/151227
-rw-r--r--tests/qemu-iotests/151.out4
-rwxr-xr-xtests/qemu-iotests/tests/stream-under-throttle121
-rw-r--r--tests/qemu-iotests/tests/stream-under-throttle.out5
4 files changed, 354 insertions, 3 deletions
diff --git a/tests/qemu-iotests/151 b/tests/qemu-iotests/151
index 93d14193d0..b4d1bc2553 100755
--- a/tests/qemu-iotests/151
+++ b/tests/qemu-iotests/151
@@ -19,7 +19,11 @@
# along with this program. If not, see <http://www.gnu.org/licenses/>.
#
+import math
import os
+import subprocess
+import time
+from typing import List, Optional
import iotests
from iotests import qemu_img
@@ -50,7 +54,7 @@ class TestActiveMirror(iotests.QMPTestCase):
self.vm = iotests.VM()
self.vm.add_drive_raw(self.vm.qmp_to_opts(blk_source))
self.vm.add_blockdev(self.vm.qmp_to_opts(blk_target))
- self.vm.add_device('virtio-blk,drive=source')
+ self.vm.add_device('virtio-blk,id=vblk,drive=source')
self.vm.launch()
def tearDown(self):
@@ -192,6 +196,227 @@ class TestActiveMirror(iotests.QMPTestCase):
self.potential_writes_in_flight = False
+class TestThrottledWithNbdExportBase(iotests.QMPTestCase):
+ image_len = 128 * 1024 * 1024 # MB
+ iops: Optional[int] = None
+ background_processes: List['subprocess.Popen[str]'] = []
+
+ def setUp(self):
+ # Must be set by subclasses
+ self.assertIsNotNone(self.iops)
+
+ qemu_img('create', '-f', iotests.imgfmt, source_img, '128M')
+ qemu_img('create', '-f', iotests.imgfmt, target_img, '128M')
+
+ self.vm = iotests.VM()
+ self.vm.launch()
+
+ result = self.vm.qmp('object-add', **{
+ 'qom-type': 'throttle-group',
+ 'id': 'thrgr',
+ 'limits': {
+ 'iops-total': self.iops,
+ 'iops-total-max': self.iops
+ }
+ })
+ self.assert_qmp(result, 'return', {})
+
+ result = self.vm.qmp('blockdev-add', **{
+ 'node-name': 'source-node',
+ 'driver': 'throttle',
+ 'throttle-group': 'thrgr',
+ 'file': {
+ 'driver': iotests.imgfmt,
+ 'file': {
+ 'driver': 'file',
+ 'filename': source_img
+ }
+ }
+ })
+ self.assert_qmp(result, 'return', {})
+
+ result = self.vm.qmp('blockdev-add', **{
+ 'node-name': 'target-node',
+ 'driver': iotests.imgfmt,
+ 'file': {
+ 'driver': 'file',
+ 'filename': target_img
+ }
+ })
+ self.assert_qmp(result, 'return', {})
+
+ self.nbd_sock = iotests.file_path('nbd.sock',
+ base_dir=iotests.sock_dir)
+ self.nbd_url = f'nbd+unix:///source-node?socket={self.nbd_sock}'
+
+ result = self.vm.qmp('nbd-server-start', addr={
+ 'type': 'unix',
+ 'data': {
+ 'path': self.nbd_sock
+ }
+ })
+ self.assert_qmp(result, 'return', {})
+
+ result = self.vm.qmp('block-export-add', id='exp0', type='nbd',
+ node_name='source-node', writable=True)
+ self.assert_qmp(result, 'return', {})
+
+ def tearDown(self):
+ # Wait for background requests to settle
+ try:
+ while True:
+ p = self.background_processes.pop()
+ while True:
+ try:
+ p.wait(timeout=0.0)
+ break
+ except subprocess.TimeoutExpired:
+ self.vm.qtest(f'clock_step {1 * 1000 * 1000 * 1000}')
+ except IndexError:
+ pass
+
+ # Cancel ongoing block jobs
+ for job in self.vm.qmp('query-jobs')['return']:
+ self.vm.qmp('block-job-cancel', device=job['id'], force=True)
+
+ while True:
+ self.vm.qtest(f'clock_step {1 * 1000 * 1000 * 1000}')
+ if len(self.vm.qmp('query-jobs')['return']) == 0:
+ break
+
+ self.vm.shutdown()
+ os.remove(source_img)
+ os.remove(target_img)
+
+
+class TestLowThrottledWithNbdExport(TestThrottledWithNbdExportBase):
+ iops = 16
+
+ def testUnderLoad(self):
+ '''
+ Throttle the source node, then issue a whole bunch of external requests
+ while the mirror job (in write-blocking mode) is running. We want to
+ see background requests being issued even while the source is under
+ full load by active writes, so that progress can be made towards READY.
+ '''
+
+ # Fill the first half of the source image; do not fill the second half,
+ # that is where we will have active requests occur. This ensures that
+ # active mirroring itself will not directly contribute to the job's
+ # progress (because when the job was started, those areas were not
+ # intended to be copied, so active mirroring will only lead to not
+ # losing progress, but also not making any).
+ self.vm.hmp_qemu_io('source-node',
+ f'aio_write -P 1 0 {self.image_len // 2}')
+ self.vm.qtest(f'clock_step {1 * 1000 * 1000 * 1000}')
+
+ # Launch the mirror job
+ mirror_buf_size = 65536
+ result = self.vm.qmp('blockdev-mirror',
+ job_id='mirror',
+ filter_node_name='mirror-node',
+ device='source-node',
+ target='target-node',
+ sync='full',
+ copy_mode='write-blocking',
+ buf_size=mirror_buf_size)
+ self.assert_qmp(result, 'return', {})
+
+ # We create the external requests via qemu-io processes on the NBD
+ # server. Have their offset start in the middle of the image so they
+ # do not overlap with the background requests (which start from the
+ # beginning).
+ active_request_offset = self.image_len // 2
+ active_request_len = 4096
+
+ # Create enough requests to saturate the node for 5 seconds
+ for _ in range(0, 5 * self.iops):
+ req = f'write -P 42 {active_request_offset} {active_request_len}'
+ active_request_offset += active_request_len
+ p = iotests.qemu_io_popen('-f', 'nbd', self.nbd_url, '-c', req)
+ self.background_processes += [p]
+
+ # Now advance the clock one I/O operation at a time by the 4 seconds
+ # (i.e. one less than 5). We expect the mirror job to issue background
+ # operations here, even though active requests are still in flight.
+ # The active requests will take precedence, however, because they have
+ # been issued earlier than mirror's background requests.
+ # Once the active requests we have started above are done (i.e. after 5
+ # virtual seconds), we expect those background requests to be worked
+ # on. We only advance 4 seconds here to avoid race conditions.
+ for _ in range(0, 4 * self.iops):
+ step = math.ceil(1 * 1000 * 1000 * 1000 / self.iops)
+ self.vm.qtest(f'clock_step {step}')
+
+ # Note how much remains to be done until the mirror job is finished
+ job_status = self.vm.qmp('query-jobs')['return'][0]
+ start_remaining = job_status['total-progress'] - \
+ job_status['current-progress']
+
+ # Create a whole bunch of more active requests
+ for _ in range(0, 10 * self.iops):
+ req = f'write -P 42 {active_request_offset} {active_request_len}'
+ active_request_offset += active_request_len
+ p = iotests.qemu_io_popen('-f', 'nbd', self.nbd_url, '-c', req)
+ self.background_processes += [p]
+
+ # Let the clock advance more. After 1 second, as noted above, we
+ # expect the background requests to be worked on. Give them a couple
+ # of seconds (specifically 4) to see their impact.
+ for _ in range(0, 5 * self.iops):
+ step = math.ceil(1 * 1000 * 1000 * 1000 / self.iops)
+ self.vm.qtest(f'clock_step {step}')
+
+ # Note how much remains to be done now. We expect this number to be
+ # reduced thanks to those background requests.
+ job_status = self.vm.qmp('query-jobs')['return'][0]
+ end_remaining = job_status['total-progress'] - \
+ job_status['current-progress']
+
+ # See that indeed progress was being made on the job, even while the
+ # node was saturated with active requests
+ self.assertGreater(start_remaining - end_remaining, 0)
+
+
+class TestHighThrottledWithNbdExport(TestThrottledWithNbdExportBase):
+ iops = 1024
+
+ def testActiveOnCreation(self):
+ '''
+ Issue requests on the mirror source node right as the mirror is
+ instated. It's possible that requests occur before the actual job is
+ created, but after the node has been put into the graph. Write
+ requests across the node must in that case be forwarded to the source
+ node without attempting to mirror them (there is no job object yet, so
+ attempting to access it would cause a segfault).
+ We do this with a lightly throttled node (i.e. quite high IOPS limit).
+ Using throttling seems to increase reproductivity, but if the limit is
+ too low, all requests allowed per second will be submitted before
+ mirror_start_job() gets to the problematic point.
+ '''
+
+ # Let qemu-img bench create write requests (enough for two seconds on
+ # the virtual clock)
+ bench_args = ['bench', '-w', '-d', '1024', '-f', 'nbd',
+ '-c', str(self.iops * 2), self.nbd_url]
+ p = iotests.qemu_tool_popen(iotests.qemu_img_args + bench_args)
+ self.background_processes += [p]
+
+ # Give qemu-img bench time to start up and issue requests
+ time.sleep(1.0)
+ # Flush the request queue, so new requests can come in right as we
+ # start blockdev-mirror
+ self.vm.qtest(f'clock_step {1 * 1000 * 1000 * 1000}')
+
+ result = self.vm.qmp('blockdev-mirror',
+ job_id='mirror',
+ device='source-node',
+ target='target-node',
+ sync='full',
+ copy_mode='write-blocking')
+ self.assert_qmp(result, 'return', {})
+
+
if __name__ == '__main__':
iotests.main(supported_fmts=['qcow2', 'raw'],
supported_protocols=['file'])
diff --git a/tests/qemu-iotests/151.out b/tests/qemu-iotests/151.out
index 89968f35d7..3f8a935a08 100644
--- a/tests/qemu-iotests/151.out
+++ b/tests/qemu-iotests/151.out
@@ -1,5 +1,5 @@
-....
+......
----------------------------------------------------------------------
-Ran 4 tests
+Ran 6 tests
OK
diff --git a/tests/qemu-iotests/tests/stream-under-throttle b/tests/qemu-iotests/tests/stream-under-throttle
new file mode 100755
index 0000000000..8d2d9e1684
--- /dev/null
+++ b/tests/qemu-iotests/tests/stream-under-throttle
@@ -0,0 +1,121 @@
+#!/usr/bin/env python3
+# group: rw
+#
+# Test streaming with throttle nodes on top
+#
+# Copyright (C) 2022 Red Hat, Inc.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+#
+
+import asyncio
+import os
+from typing import List
+import iotests
+from iotests import qemu_img_create, qemu_io
+
+
+image_size = 256 * 1024 * 1024
+base_img = os.path.join(iotests.test_dir, 'base.img')
+top_img = os.path.join(iotests.test_dir, 'top.img')
+
+
+class TcgVM(iotests.VM):
+ '''
+ Variant of iotests.VM that uses -accel tcg. Simply using
+ iotests.VM.add_args('-accel', 'tcg') is not sufficient, because that will
+ put -accel qtest before -accel tcg, and -accel arguments are prioritized in
+ the order they appear.
+ '''
+ @property
+ def _base_args(self) -> List[str]:
+ # Put -accel tcg first so it takes precedence
+ return ['-accel', 'tcg'] + super()._base_args
+
+
+class TestStreamWithThrottle(iotests.QMPTestCase):
+ def setUp(self) -> None:
+ '''
+ Create a simple backing chain between two images, write something to
+ the base image. Attach them to the VM underneath two throttle nodes,
+ one of which has actually no limits set, but the other does. Then put
+ a virtio-blk device on top.
+ This test configuration has been taken from
+ https://gitlab.com/qemu-project/qemu/-/issues/1215
+ '''
+ qemu_img_create('-f', iotests.imgfmt, base_img, str(image_size))
+ qemu_img_create('-f', iotests.imgfmt, '-b', base_img, '-F',
+ iotests.imgfmt, top_img, str(image_size))
+
+ # Write something to stream
+ qemu_io(base_img, '-c', f'write 0 {image_size}')
+
+ blockdev = {
+ 'driver': 'throttle',
+ 'node-name': 'throttled-node',
+ 'throttle-group': 'thrgr-limited',
+ 'file': {
+ 'driver': 'throttle',
+ 'throttle-group': 'thrgr-unlimited',
+ 'file': {
+ 'driver': iotests.imgfmt,
+ 'node-name': 'unthrottled-node',
+ 'file': {
+ 'driver': 'file',
+ 'filename': top_img
+ }
+ }
+ }
+ }
+
+ # Issue 1215 is not reproducible in qtest mode, which is why we need to
+ # create an -accel tcg VM
+ self.vm = TcgVM()
+ self.vm.add_object('iothread,id=iothr0')
+ self.vm.add_object('throttle-group,id=thrgr-unlimited')
+ self.vm.add_object('throttle-group,id=thrgr-limited,'
+ 'x-iops-total=10000,x-bps-total=104857600')
+ self.vm.add_blockdev(self.vm.qmp_to_opts(blockdev))
+ self.vm.add_device('virtio-blk,iothread=iothr0,drive=throttled-node')
+ self.vm.launch()
+
+ def tearDown(self) -> None:
+ self.vm.shutdown()
+ os.remove(top_img)
+ os.remove(base_img)
+
+ def test_stream(self) -> None:
+ '''
+ Do a simple stream beneath the two throttle nodes. Should complete
+ with no problems.
+ '''
+ result = self.vm.qmp('block-stream',
+ job_id='stream',
+ device='unthrottled-node')
+ self.assert_qmp(result, 'return', {})
+
+ # Should succeed and not time out
+ try:
+ self.vm.run_job('stream')
+ except asyncio.TimeoutError:
+ # VM may be stuck, kill it before tearDown()
+ self.vm.kill()
+ raise
+
+
+if __name__ == '__main__':
+ # Must support backing images
+ iotests.main(supported_fmts=['qcow', 'qcow2', 'qed'],
+ supported_protocols=['file'],
+ required_fmts=['throttle'])
diff --git a/tests/qemu-iotests/tests/stream-under-throttle.out b/tests/qemu-iotests/tests/stream-under-throttle.out
new file mode 100644
index 0000000000..ae1213e6f8
--- /dev/null
+++ b/tests/qemu-iotests/tests/stream-under-throttle.out
@@ -0,0 +1,5 @@
+.
+----------------------------------------------------------------------
+Ran 1 tests
+
+OK