From dbfa29bce1e206eb049504312b8eadf6fa92c61f Mon Sep 17 00:00:00 2001 From: Simon Rettberg Date: Wed, 26 Jul 2023 16:22:15 +0200 Subject: [libvirt] Add support for CPU pinning, honor host's SMT topology Read the system's SMT topology, and apply it to the guest via CPU pinning. In qemu, sibling threads on the same CPU core are adjacent regarding the vCPU IDs, so make sure we assign them in ascending order. --- .../vmchooser/plugins/qemukvm/run-virt.include | 14 +++++- .../runvirt/plugin/qemu/cmdln/CommandLineArgs.java | 47 ++++++++++++++++++++ .../configuration/TransformationGenericCpu.java | 51 +++++++++++++++++----- .../TransformationGenericCpuTest.java | 4 +- 4 files changed, 101 insertions(+), 15 deletions(-) (limited to 'core/modules/qemu') diff --git a/core/modules/qemu/data/opt/openslx/vmchooser/plugins/qemukvm/run-virt.include b/core/modules/qemu/data/opt/openslx/vmchooser/plugins/qemukvm/run-virt.include index ecf68e0c..b15015ee 100644 --- a/core/modules/qemu/data/opt/openslx/vmchooser/plugins/qemukvm/run-virt.include +++ b/core/modules/qemu/data/opt/openslx/vmchooser/plugins/qemukvm/run-virt.include @@ -74,7 +74,13 @@ run_plugin() { fi # set device passthrough debug mode - debug_pth="false" + local debug_pth="false" + + # Use cat here instead of redirect because of globbing + local cputhreads + cputhreads="$( cat /sys/devices/system/cpu/cpu*/topology/core_cpus_list | awk '!a[$1]{if(b)printf";";printf $1;a[$1]=1;b=1}' )" + # Try legacy name + [ -z "$cputhreads" ] && cputhreads="$( cat /sys/devices/system/cpu/cpu*/topology/thread_siblings_list | awk '!a[$1]{if(b)printf";";printf $1;a[$1]=1;b=1}' )" # call the Libvirt Java tool to finalize configuration and start VM declare -rg VIRTCMD="java" @@ -89,7 +95,11 @@ run_plugin() { notempty TMPCONFIG && VIRTCMDOPTS+=( "-vmcfginp" "${TMPCONFIG}" ) notempty vm_final_config && VIRTCMDOPTS+=( "-vmcfgout" "${vm_final_config}" ) notempty IMGUUID && VIRTCMDOPTS+=( "-vmuuid" "${IMGUUID}" ) - notempty HW_CORES && VIRTCMDOPTS+=( "-vmncpus" "${HW_CORES}" ) + if notempty cputhreads; then + VIRTCMDOPTS+=( "-cputopo" "${cputhreads}" ) + elif notempty HW_THREADS; then + VIRTCMDOPTS+=( "-vmncpus" "${HW_THREADS}" ) + fi notempty VM_MEM && VIRTCMDOPTS+=( "-vmmem" "${VM_MEM}" ) notempty VM_MAC_ADDR && VIRTCMDOPTS+=( "-vmmac0" "${VM_MAC_ADDR}" ) notempty vm_diskfile && VIRTCMDOPTS+=( "-vmhdd0" "${vm_diskfile}" ) diff --git a/core/modules/qemu/runvirt-plugin-qemu/src/main/java/org/openslx/runvirt/plugin/qemu/cmdln/CommandLineArgs.java b/core/modules/qemu/runvirt-plugin-qemu/src/main/java/org/openslx/runvirt/plugin/qemu/cmdln/CommandLineArgs.java index 1ab99076..21b11968 100644 --- a/core/modules/qemu/runvirt-plugin-qemu/src/main/java/org/openslx/runvirt/plugin/qemu/cmdln/CommandLineArgs.java +++ b/core/modules/qemu/runvirt-plugin-qemu/src/main/java/org/openslx/runvirt/plugin/qemu/cmdln/CommandLineArgs.java @@ -11,6 +11,9 @@ import org.apache.commons.cli.HelpFormatter; import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.openslx.util.Util; /** * Command line argument parser for the run-virt QEMU plugin (command line tool). @@ -20,6 +23,11 @@ import org.apache.commons.cli.ParseException; */ public class CommandLineArgs { + /** + * Instance of a logger to log messages. + */ + private static final Logger LOGGER = LogManager.getLogger( CommandLineArgs.class ); + /** * Parser for parsing command line arguments. */ @@ -258,6 +266,43 @@ public class CommandLineArgs return numCpus; } + public List> getCpuTopology() + { + String arg = this.getArgument( CmdLnOption.VM_CPU_TOPO ); + if ( Util.isEmptyString( arg ) ) + return null; + String[] scores = arg.split( ";" ); + List> retval = new ArrayList<>( scores.length ); + for ( int c = 0; c < scores.length; ++c ) { + if ( Util.isEmptyString( scores[c] ) ) { + LOGGER.warn( "Could not parse CPU topology: empty group element" ); + return null; + } + String[] coreThreads = scores[c].split( "," ); + ArrayList current = new ArrayList<>(); + retval.add( current ); + for ( int t = 0; t < coreThreads.length; ++t ) { + int from, to; + String[] fromTo = coreThreads[t].split( "-" ); + if ( fromTo.length == 0 || fromTo.length > 2 + || Util.isEmptyString( fromTo[0] ) || ( fromTo.length > 1 && Util.isEmptyString( fromTo[1] ) ) ) { + LOGGER.warn( "Could not parse CPU topology: empty or malformed sibling element '" + coreThreads[t] + "'" ); + return null; + } + from = Util.parseInt( fromTo[0], -1 ); + to = fromTo.length == 2 ? Util.parseInt( fromTo[1], -1 ) : from; + if ( from == -1 || to == -1 || to < from ) { + LOGGER.warn( "Could not parse CPU topology sibling number '" + coreThreads[t] + "' from '" + scores[c] + "'" ); + return null; + } + for ( int i = from; i <= to; ++i ) { + current.add( i ); + } + } + } + return retval; + } + /** * Returns the argument of the command line option {@link CmdLnOption#VM_MEM}. * @@ -457,6 +502,8 @@ public class CommandLineArgs // @formatter:off XML_EDIT ( '0', "xmledit", 0, "Spawn a text editor with the final XML before starting, so it can be edited" + " for testing and debugging purposes"), + VM_CPU_TOPO ( '1', "cputopo", 1, "Set pairs of CPUs belonging to the same thread, semi-colon separated." + + " Each group can contain commas or dashes to mark ranges. E.g. 0,1;2-3;4;5;6;7;8,9,10,11" ), VM_MAC0 ( 'a', "vmmac0", 1, "MAC address for the first network interface" ), DEBUG ( 'b', "debug", 1, "Enable or disable debug mode" ), VM_NCPUS ( 'c', "vmncpus", 1, "Number of virtual CPUs for the virtual machine" ), diff --git a/core/modules/qemu/runvirt-plugin-qemu/src/main/java/org/openslx/runvirt/plugin/qemu/configuration/TransformationGenericCpu.java b/core/modules/qemu/runvirt-plugin-qemu/src/main/java/org/openslx/runvirt/plugin/qemu/configuration/TransformationGenericCpu.java index 73cd4c1f..fdeef511 100644 --- a/core/modules/qemu/runvirt-plugin-qemu/src/main/java/org/openslx/runvirt/plugin/qemu/configuration/TransformationGenericCpu.java +++ b/core/modules/qemu/runvirt-plugin-qemu/src/main/java/org/openslx/runvirt/plugin/qemu/configuration/TransformationGenericCpu.java @@ -1,5 +1,7 @@ package org.openslx.runvirt.plugin.qemu.configuration; +import java.util.List; + import org.openslx.libvirt.domain.Domain; import org.openslx.libvirt.domain.Domain.CpuCheck; import org.openslx.libvirt.domain.Domain.CpuMode; @@ -30,11 +32,6 @@ public class TransformationGenericCpu extends TransformationGeneric 0!" ); + } else if ( args.getVmNumCpus() < 1 && args.getCpuTopology() == null ) { + throw new TransformationException( "Invalid number of virtual CPUs or CPU topology specified! Expected a number n > 0!" ); } } @@ -67,17 +64,49 @@ public class TransformationGenericCpu extends TransformationGeneric> topo = args.getCpuTopology(); + boolean onlyOneThread = false; + int numCores; + + if ( topo == null ) { + numCores = args.getVmNumCpus(); + } else { + int last = -1; + for ( List group : topo ) { + if ( last != -1 && last != group.size() ) { + onlyOneThread = true; + break; + } + last = group.size(); + } + numCores = topo.size(); + } // set detailed CPU topology config.setCpuDies( TransformationGenericCpu.CPU_NUM_DIES ); config.setCpuSockets( TransformationGenericCpu.CPU_NUM_SOCKETS ); - config.setCpuCores( args.getVmNumCpus() ); - config.setCpuThreads( TransformationGenericCpu.CPU_NUM_THREADS ); + config.setCpuCores( numCores ); + config.setCpuThreads( ( topo == null || onlyOneThread ) ? 1 : topo.get( 0 ).size() ); // set maximum allocated CPUs for the VM final int maxCpus = TransformationGenericCpu.CPU_NUM_DIES * TransformationGenericCpu.CPU_NUM_SOCKETS - * args.getVmNumCpus() * TransformationGenericCpu.CPU_NUM_THREADS; + * numCores * config.getCpuThreads(); config.setVCpu( maxCpus ); + config.setCpuMigratable( false ); + + // Set CPU pinning if known + config.resetCpuPin(); + if ( topo != null ) { + int guestCore = 0; + for ( List group : topo ) { + for ( int hostCore : group ) { + config.addCpuPin( guestCore++, hostCore ); + if ( onlyOneThread ) + break; + } + } + } } } diff --git a/core/modules/qemu/runvirt-plugin-qemu/src/test/java/org/openslx/runvirt/plugin/qemu/configuration/TransformationGenericCpuTest.java b/core/modules/qemu/runvirt-plugin-qemu/src/test/java/org/openslx/runvirt/plugin/qemu/configuration/TransformationGenericCpuTest.java index f90c5625..91308fab 100644 --- a/core/modules/qemu/runvirt-plugin-qemu/src/test/java/org/openslx/runvirt/plugin/qemu/configuration/TransformationGenericCpuTest.java +++ b/core/modules/qemu/runvirt-plugin-qemu/src/test/java/org/openslx/runvirt/plugin/qemu/configuration/TransformationGenericCpuTest.java @@ -30,12 +30,12 @@ public class TransformationGenericCpuTest transformation.transform( config, args ); assertEquals( CpuMode.HOST_PASSTHROUGH, config.getCpuMode() ); - assertEquals( CpuCheck.PARTIAL, config.getCpuCheck() ); + assertEquals( CpuCheck.NONE, config.getCpuCheck() ); final int numDies = TransformationGenericCpu.CPU_NUM_DIES; final int numSockets = TransformationGenericCpu.CPU_NUM_SOCKETS; final int numCores = Integer.valueOf( TransformationTestUtils.DEFAULT_VM_NCPUS ); - final int numThreads = TransformationGenericCpu.CPU_NUM_THREADS; + final int numThreads = 1; final int numVCpus = numDies * numSockets * numCores * numThreads; -- cgit v1.2.3-55-g7522