[libvirt] Add support for CPU pinning, honor host's SMT topology

Read the system's SMT topology, and apply it to the guest via CPU
pinning. In qemu, sibling threads on the same CPU core are adjacent
regarding the vCPU IDs, so make sure we assign them in ascending order.

4 files changed, 101 insertions, 15 deletions

diff --git a/core/modules/qemu/data/opt/openslx/vmchooser/plugins/qemukvm/run-virt.include b/core/modules/qemu/data/opt/openslx/vmchooser/plugins/qemukvm/run-virt.include
index ecf68e0c..b15015ee 100644
--- a/core/modules/qemu/data/opt/openslx/vmchooser/plugins/qemukvm/run-virt.include
+++ b/core/modules/qemu/data/opt/openslx/vmchooser/plugins/qemukvm/run-virt.include
@@ -74,7 +74,13 @@ run_plugin() {
 	fi
 
 	# set device passthrough debug mode
-	debug_pth="false"
+	local debug_pth="false"
+
+	# Use cat here instead of redirect because of globbing
+	local cputhreads
+	cputhreads="$( cat /sys/devices/system/cpu/cpu*/topology/core_cpus_list | awk '!a[$1]{if(b)printf";";printf $1;a[$1]=1;b=1}' )"
+	# Try legacy name
+	[ -z "$cputhreads" ] && cputhreads="$( cat /sys/devices/system/cpu/cpu*/topology/thread_siblings_list | awk '!a[$1]{if(b)printf";";printf $1;a[$1]=1;b=1}' )"
 
 	# call the Libvirt Java tool to finalize configuration and start VM
 	declare -rg VIRTCMD="java"
@@ -89,7 +95,11 @@ run_plugin() {
 	notempty TMPCONFIG         && VIRTCMDOPTS+=( "-vmcfginp"    "${TMPCONFIG}" )
 	notempty vm_final_config   && VIRTCMDOPTS+=( "-vmcfgout"    "${vm_final_config}" )
 	notempty IMGUUID           && VIRTCMDOPTS+=( "-vmuuid"      "${IMGUUID}" )
-	notempty HW_CORES          && VIRTCMDOPTS+=( "-vmncpus"     "${HW_CORES}" )
+	if notempty cputhreads; then
+		VIRTCMDOPTS+=( "-cputopo"     "${cputhreads}" )
+	elif notempty HW_THREADS; then
+		VIRTCMDOPTS+=( "-vmncpus"     "${HW_THREADS}" )
+	fi
 	notempty VM_MEM            && VIRTCMDOPTS+=( "-vmmem"       "${VM_MEM}" )
 	notempty VM_MAC_ADDR       && VIRTCMDOPTS+=( "-vmmac0"      "${VM_MAC_ADDR}" )
 	notempty vm_diskfile       && VIRTCMDOPTS+=( "-vmhdd0"      "${vm_diskfile}" )
diff --git a/core/modules/qemu/runvirt-plugin-qemu/src/main/java/org/openslx/runvirt/plugin/qemu/cmdln/CommandLineArgs.java b/core/modules/qemu/runvirt-plugin-qemu/src/main/java/org/openslx/runvirt/plugin/qemu/cmdln/CommandLineArgs.java
index 1ab99076..21b11968 100644
--- a/core/modules/qemu/runvirt-plugin-qemu/src/main/java/org/openslx/runvirt/plugin/qemu/cmdln/CommandLineArgs.java
+++ b/core/modules/qemu/runvirt-plugin-qemu/src/main/java/org/openslx/runvirt/plugin/qemu/cmdln/CommandLineArgs.java
@@ -11,6 +11,9 @@ import org.apache.commons.cli.HelpFormatter;
 import org.apache.commons.cli.Option;
 import org.apache.commons.cli.Options;
 import org.apache.commons.cli.ParseException;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.openslx.util.Util;
 
 /**
  * Command line argument parser for the run-virt QEMU plugin (command line tool).
@@ -21,6 +24,11 @@ import org.apache.commons.cli.ParseException;
 public class CommandLineArgs
 {
 	/**
+	 * Instance of a logger to log messages.
+	 */
+	private static final Logger LOGGER = LogManager.getLogger( CommandLineArgs.class );
+
+	/**
 	 * Parser for parsing command line arguments.
 	 */
 	private CommandLineParser cmdLnParser = null;
@@ -258,6 +266,43 @@ public class CommandLineArgs
 		return numCpus;
 	}
 
+	public List<List<Integer>> getCpuTopology()
+	{
+		String arg = this.getArgument( CmdLnOption.VM_CPU_TOPO );
+		if ( Util.isEmptyString( arg ) )
+			return null;
+		String[] scores = arg.split( ";" );
+		List<List<Integer>> retval = new ArrayList<>( scores.length );
+		for ( int c = 0; c < scores.length; ++c ) {
+			if ( Util.isEmptyString( scores[c] ) ) {
+				LOGGER.warn( "Could not parse CPU topology: empty group element" );
+				return null;
+			}
+			String[] coreThreads = scores[c].split( "," );
+			ArrayList<Integer> current = new ArrayList<>();
+			retval.add( current );
+			for ( int t = 0; t < coreThreads.length; ++t ) {
+				int from, to;
+				String[] fromTo = coreThreads[t].split( "-" );
+				if ( fromTo.length == 0 || fromTo.length > 2
+						|| Util.isEmptyString( fromTo[0] ) || ( fromTo.length > 1 && Util.isEmptyString( fromTo[1] ) ) ) {
+					LOGGER.warn( "Could not parse CPU topology: empty or malformed sibling element '" + coreThreads[t] + "'" );
+					return null;
+				}
+				from = Util.parseInt( fromTo[0], -1 );
+				to = fromTo.length == 2 ? Util.parseInt( fromTo[1], -1 ) : from;
+				if ( from == -1 || to == -1 || to < from ) {
+					LOGGER.warn( "Could not parse CPU topology sibling number '" + coreThreads[t] + "' from '" + scores[c] + "'" );
+					return null;
+				}
+				for ( int i = from; i <= to; ++i ) {
+					current.add( i );
+				}
+			}
+		}
+		return retval;
+	}
+
 	/**
 	 * Returns the argument of the command line option {@link CmdLnOption#VM_MEM}.
 	 * 
@@ -457,6 +502,8 @@ public class CommandLineArgs
 		// @formatter:off
 		XML_EDIT    ( '0', "xmledit",     0, "Spawn a text editor with the final XML before starting, so it can be edited"
 		                                   + " for testing and debugging purposes"),
+		VM_CPU_TOPO ( '1', "cputopo",     1, "Set pairs of CPUs belonging to the same thread, semi-colon separated."
+		                                   + " Each group can contain commas or dashes to mark ranges. E.g. 0,1;2-3;4;5;6;7;8,9,10,11" ),
 		VM_MAC0     ( 'a', "vmmac0",      1, "MAC address for the first network interface" ),
 		DEBUG       ( 'b', "debug",       1, "Enable or disable debug mode" ),
 		VM_NCPUS    ( 'c', "vmncpus",     1, "Number of virtual CPUs for the virtual machine" ),
diff --git a/core/modules/qemu/runvirt-plugin-qemu/src/main/java/org/openslx/runvirt/plugin/qemu/configuration/TransformationGenericCpu.java b/core/modules/qemu/runvirt-plugin-qemu/src/main/java/org/openslx/runvirt/plugin/qemu/configuration/TransformationGenericCpu.java
index 73cd4c1f..fdeef511 100644
--- a/core/modules/qemu/runvirt-plugin-qemu/src/main/java/org/openslx/runvirt/plugin/qemu/configuration/TransformationGenericCpu.java
+++ b/core/modules/qemu/runvirt-plugin-qemu/src/main/java/org/openslx/runvirt/plugin/qemu/configuration/TransformationGenericCpu.java
@@ -1,5 +1,7 @@
 package org.openslx.runvirt.plugin.qemu.configuration;
 
+import java.util.List;
+
 import org.openslx.libvirt.domain.Domain;
 import org.openslx.libvirt.domain.Domain.CpuCheck;
 import org.openslx.libvirt.domain.Domain.CpuMode;
@@ -31,11 +33,6 @@ public class TransformationGenericCpu extends TransformationGeneric<Domain, Comm
 	public static final int CPU_NUM_SOCKETS = 1;
 
 	/**
-	 * Number of threads per virtual machine's CPU.
-	 */
-	public static final int CPU_NUM_THREADS = 1;
-
-	/**
 	 * Creates a new generic CPU transformation for Libvirt/QEMU virtualization configurations.
 	 */
 	public TransformationGenericCpu()
@@ -54,8 +51,8 @@ public class TransformationGenericCpu extends TransformationGeneric<Domain, Comm
 	{
 		if ( config == null || args == null ) {
 			throw new TransformationException( "Virtualization configuration or input arguments are missing!" );
-		} else if ( args.getVmNumCpus() < 1 ) {
-			throw new TransformationException( "Invalid number of virtual CPUs specified! Expected a number n > 0!" );
+		} else if ( args.getVmNumCpus() < 1 && args.getCpuTopology() == null ) {
+			throw new TransformationException( "Invalid number of virtual CPUs or CPU topology specified! Expected a number n > 0!" );
 		}
 	}
 
@@ -67,17 +64,49 @@ public class TransformationGenericCpu extends TransformationGeneric<Domain, Comm
 
 		// set general CPU modes
 		config.setCpuMode( CpuMode.HOST_PASSTHROUGH );
-		config.setCpuCheck( CpuCheck.PARTIAL );
+		config.setCpuCheck( CpuCheck.NONE );
+
+		List<List<Integer>> topo = args.getCpuTopology();
+		boolean onlyOneThread = false;
+		int numCores;
+
+		if ( topo == null ) {
+			numCores = args.getVmNumCpus();
+		} else {
+			int last = -1;
+			for ( List<Integer> group : topo ) {
+				if ( last != -1 && last != group.size() ) {
+					onlyOneThread = true;
+					break;
+				}
+				last = group.size();
+			}
+			numCores = topo.size();
+		}
 
 		// set detailed CPU topology
 		config.setCpuDies( TransformationGenericCpu.CPU_NUM_DIES );
 		config.setCpuSockets( TransformationGenericCpu.CPU_NUM_SOCKETS );
-		config.setCpuCores( args.getVmNumCpus() );
-		config.setCpuThreads( TransformationGenericCpu.CPU_NUM_THREADS );
+		config.setCpuCores( numCores );
+		config.setCpuThreads( ( topo == null || onlyOneThread ) ? 1 : topo.get( 0 ).size() );
 
 		// set maximum allocated CPUs for the VM
 		final int maxCpus = TransformationGenericCpu.CPU_NUM_DIES * TransformationGenericCpu.CPU_NUM_SOCKETS
-				* args.getVmNumCpus() * TransformationGenericCpu.CPU_NUM_THREADS;
+				* numCores * config.getCpuThreads();
 		config.setVCpu( maxCpus );
+		config.setCpuMigratable( false );
+		
+		// Set CPU pinning if known
+		config.resetCpuPin();
+		if ( topo != null ) {
+			int guestCore = 0;
+			for ( List<Integer> group : topo ) {
+				for ( int hostCore : group ) {
+					config.addCpuPin( guestCore++, hostCore );
+					if ( onlyOneThread )
+						break;
+				}
+			}
+		}
 	}
 }
diff --git a/core/modules/qemu/runvirt-plugin-qemu/src/test/java/org/openslx/runvirt/plugin/qemu/configuration/TransformationGenericCpuTest.java b/core/modules/qemu/runvirt-plugin-qemu/src/test/java/org/openslx/runvirt/plugin/qemu/configuration/TransformationGenericCpuTest.java
index f90c5625..91308fab 100644
--- a/core/modules/qemu/runvirt-plugin-qemu/src/test/java/org/openslx/runvirt/plugin/qemu/configuration/TransformationGenericCpuTest.java
+++ b/core/modules/qemu/runvirt-plugin-qemu/src/test/java/org/openslx/runvirt/plugin/qemu/configuration/TransformationGenericCpuTest.java
@@ -30,12 +30,12 @@ public class TransformationGenericCpuTest
 		transformation.transform( config, args );
 
 		assertEquals( CpuMode.HOST_PASSTHROUGH, config.getCpuMode() );
-		assertEquals( CpuCheck.PARTIAL, config.getCpuCheck() );
+		assertEquals( CpuCheck.NONE, config.getCpuCheck() );
 
 		final int numDies = TransformationGenericCpu.CPU_NUM_DIES;
 		final int numSockets = TransformationGenericCpu.CPU_NUM_SOCKETS;
 		final int numCores = Integer.valueOf( TransformationTestUtils.DEFAULT_VM_NCPUS );
-		final int numThreads = TransformationGenericCpu.CPU_NUM_THREADS;
+		final int numThreads = 1;
 
 		final int numVCpus = numDies * numSockets * numCores * numThreads;