From f7d38bc10b11abadbcd9b12b1784d7108f0a5d7e Mon Sep 17 00:00:00 2001 From: Manuel Bentele Date: Thu, 10 Jun 2021 10:47:03 +0200 Subject: [qemu] Implement passthrough of NVIDIA GPUs The implementation adds specified PCI devics of a NVIDIA GPU on the host system to the final Libvirt domain XML configuration for a NVIDIA GPU passthrough. In addition to that, the implementation adds support for the Looking Glass Client to display the framebuffer of the NVIDIA GPU on the host system through a shared memory device. --- .../java/org/openslx/runvirt/plugin/qemu/App.java | 32 ++-- .../runvirt/plugin/qemu/cmdln/CommandLineArgs.java | 120 ++++++++++---- .../TransformationSpecificQemuArchitecture.java | 27 ++-- ...sformationSpecificQemuGpuPassthroughNvidia.java | 179 ++++++++++++++++++++- .../runvirt/viewer/ViewerLookingGlassClient.java | 105 ++++++++++++ .../org/openslx/runvirt/plugin/qemu/AppTest.java | 8 +- .../plugin/qemu/cmdln/CommandLineArgsTest.java | 54 +++++++ ...mationSpecificQemuGpuPassthroughNvidiaTest.java | 129 +++++++++++++++ .../configuration/TransformationTestUtils.java | 7 +- 9 files changed, 592 insertions(+), 69 deletions(-) create mode 100644 core/modules/qemu/runvirt-plugin-qemu/src/main/java/org/openslx/runvirt/viewer/ViewerLookingGlassClient.java create mode 100644 core/modules/qemu/runvirt-plugin-qemu/src/test/java/org/openslx/runvirt/plugin/qemu/configuration/TransformationSpecificQemuGpuPassthroughNvidiaTest.java diff --git a/core/modules/qemu/runvirt-plugin-qemu/src/main/java/org/openslx/runvirt/plugin/qemu/App.java b/core/modules/qemu/runvirt-plugin-qemu/src/main/java/org/openslx/runvirt/plugin/qemu/App.java index a3e1c5b7..5ea7b720 100644 --- a/core/modules/qemu/runvirt-plugin-qemu/src/main/java/org/openslx/runvirt/plugin/qemu/App.java +++ b/core/modules/qemu/runvirt-plugin-qemu/src/main/java/org/openslx/runvirt/plugin/qemu/App.java @@ -1,6 +1,7 @@ package org.openslx.runvirt.plugin.qemu; import java.io.File; +import java.util.Arrays; import org.apache.log4j.BasicConfigurator; import org.apache.log4j.LogManager; @@ -29,6 +30,7 @@ import org.openslx.runvirt.plugin.qemu.virtualization.LibvirtHypervisorQemu; import org.openslx.runvirt.plugin.qemu.virtualization.LibvirtHypervisorQemu.QemuSessionType; import org.openslx.runvirt.viewer.Viewer; import org.openslx.runvirt.viewer.ViewerException; +import org.openslx.runvirt.viewer.ViewerLookingGlassClient; import org.openslx.runvirt.viewer.ViewerVirtManager; import org.openslx.runvirt.viewer.ViewerVirtViewer; import org.openslx.runvirt.virtualization.LibvirtHypervisor; @@ -185,12 +187,18 @@ public class App // create specific viewer to display Libvirt VM final Viewer vmViewer; - if ( cmdLn.isDebugEnabled() ) { - // create specific Virtual Machine Manager viewer if debug mode is enabled - vmViewer = new ViewerVirtManager( vm, hypervisor ); + if ( cmdLn.isNvidiaGpuPassthroughEnabled() ) { + // viewer for GPU passthrough (framebuffer access) is required + vmViewer = new ViewerLookingGlassClient( vm, hypervisor, cmdLn.isDebugEnabled() ); } else { - // create Virtual Viewer if debug mode is disabled - vmViewer = new ViewerVirtViewer( vm, hypervisor ); + // viewer for non-GPU passthrough (no framebuffer access) is required + if ( cmdLn.isDebugEnabled() ) { + // create specific Virtual Machine Manager viewer if debug mode is enabled + vmViewer = new ViewerVirtManager( vm, hypervisor ); + } else { + // create Virtual Viewer if debug mode is disabled + vmViewer = new ViewerVirtViewer( vm, hypervisor ); + } } // display Libvirt VM with the specific viewer on the screen @@ -258,16 +266,20 @@ public class App for ( CmdLnOption option : CmdLnOption.values() ) { final String paddedLongOption = String.format( "%-" + longOptionLengthMax + "s", option.getLongOption() ); - final String longOptionArgument; + String[] longOptionArguments; // only request and log argument if option has an command line argument - if ( option.hasArgument() ) { - longOptionArgument = cmdLn.getArgument( option ); + if ( option.getNumArguments() > 0 ) { + longOptionArguments = cmdLn.getArguments( option ); + + if ( longOptionArguments == null ) { + longOptionArguments = new String[] { "no argument specified" }; + } } else { - longOptionArgument = new String( "[option has no argument]" ); + longOptionArguments = new String[] { "option has no argument" }; } - LOGGER.debug( "\t" + paddedLongOption + ": " + longOptionArgument ); + LOGGER.debug( "\t" + paddedLongOption + ": " + Arrays.toString( longOptionArguments ) ); } } } diff --git a/core/modules/qemu/runvirt-plugin-qemu/src/main/java/org/openslx/runvirt/plugin/qemu/cmdln/CommandLineArgs.java b/core/modules/qemu/runvirt-plugin-qemu/src/main/java/org/openslx/runvirt/plugin/qemu/cmdln/CommandLineArgs.java index 1fe342b1..589dd197 100644 --- a/core/modules/qemu/runvirt-plugin-qemu/src/main/java/org/openslx/runvirt/plugin/qemu/cmdln/CommandLineArgs.java +++ b/core/modules/qemu/runvirt-plugin-qemu/src/main/java/org/openslx/runvirt/plugin/qemu/cmdln/CommandLineArgs.java @@ -1,9 +1,14 @@ package org.openslx.runvirt.plugin.qemu.cmdln; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.CommandLineParser; import org.apache.commons.cli.DefaultParser; import org.apache.commons.cli.HelpFormatter; +import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; import org.apache.commons.cli.ParseException; @@ -71,8 +76,14 @@ public class CommandLineArgs private void createCmdLnOptions() { for ( CmdLnOption option : CmdLnOption.values() ) { - this.cmdLnOptions.addOption( option.getShortOption(), option.getLongOption(), option.hasArgument(), - option.getDescription() ); + final Option cmdlnOption; + + final boolean hasArg = ( option.getNumArguments() > 0 ) ? true : false; + cmdlnOption = new Option( option.getShortOption(), option.getLongOption(), hasArg, option.getDescription() ); + cmdlnOption.setValueSeparator( ',' ); + cmdlnOption.setArgs( option.getNumArguments() ); + + this.cmdLnOptions.addOption( cmdlnOption ); } } @@ -117,6 +128,17 @@ public class CommandLineArgs return this.cmdLn.getOptionValue( cmdLnOption.getShortOption() ); } + /** + * Returns the parsed arguments of the specified command line option. + * + * @param cmdLnOption command line option for that the parsed arguments should be returned. + * @return parsed argument of the command line option. + */ + public String[] getArguments( CmdLnOption cmdLnOption ) + { + return this.cmdLn.getOptionValues( cmdLnOption.getShortOption() ); + } + /** * Returns the presence of the command line option {@link CmdLnOption#HELP}. * @@ -345,6 +367,35 @@ public class CommandLineArgs return this.getArgument( CmdLnOption.VM_FSTGT1 ); } + /** + * Returns the argument of the command line option {@link CmdLnOption#VM_NVGPUIDS0}. + * + * @return argument of the command line option {@link CmdLnOption#VM_NVGPUIDS0}. + */ + public List getVmNvGpuIds0() + { + final String[] nvidiaPciIdsRaw = this.getArguments( CmdLnOption.VM_NVGPUIDS0 ); + final ArrayList nvidiaPciIds; + + if ( nvidiaPciIdsRaw == null || nvidiaPciIdsRaw.length <= 0 ) { + nvidiaPciIds = new ArrayList(); + } else { + nvidiaPciIds = new ArrayList( Arrays.asList( nvidiaPciIdsRaw ) ); + } + + return nvidiaPciIds; + } + + /** + * Returns the state whether a passthrough of a NVIDIA GPU is required. + * + * @return state whether a passthrough of a NVIDIA GPU is required. + */ + public boolean isNvidiaGpuPassthroughEnabled() + { + return this.getVmNvGpuIds0().size() > 0; + } + /** * Command line options for the run-virt QEMU plugin (command line tool). * @@ -354,28 +405,31 @@ public class CommandLineArgs public enum CmdLnOption { // @formatter:off - HELP ( 'h', "help", false, "" ), - DEBUG ( 'b', "debug", true, "Enable or disable debug mode" ), - VM_CFGINP ( 'i', "vmcfginp", true, "File name of an existing and filtered Libvirt domain XML configuration file" ), - VM_CFGOUT ( 'o', "vmcfgout", true, "File name to output a finalized Libvirt domain XML configuration file" ), - VM_NAME ( 'n', "vmname", true, "Name for the virtual machine" ), - VM_UUID ( 'u', "vmuuid", true, "UUID for the virtual machine" ), - VM_DSPLNAME ( 'd', "vmdsplname", true, "Display name for the virtual machine" ), - VM_OS ( 's', "vmos", true, "Operating system running in the virtual machine" ), - VM_NCPUS ( 'c', "vmncpus", true, "Number of virtual CPUs for the virtual machine" ), - VM_MEM ( 'm', "vmmem", true, "Amount of memory for the virtual machine" ), - VM_HDD0 ( 'r', "vmhdd0", true, "Disk image for the first HDD device" ), - VM_FLOPPY0 ( 'f', "vmfloppy0", true, "Disk image for the first floppy drive" ), - VM_FLOPPY1 ( 'g', "vmfloppy1", true, "Disk image for the second floppy drive" ), - VM_CDROM0 ( 'k', "vmcdrom0", true, "Disk image for the first CDROM drive" ), - VM_CDROM1 ( 'l', "vmcdrom1", true, "Disk image for the second CDROM drive" ), - VM_PARALLEL0( 'p', "vmparallel0", true, "Device for the first parallel port interface" ), - VM_SERIAL0 ( 'q', "vmserial0", true, "Device for the first serial port interface" ), - VM_MAC0 ( 'a', "vmmac0", true, "MAC address for the first network interface" ), - VM_FSSRC0 ( 't', "vmfssrc0", true, "Source directory for first file system passthrough (shared folder)" ), - VM_FSTGT0 ( 'e', "vmfstgt0", true, "Target directory for first file system passthrough (shared folder)" ), - VM_FSSRC1 ( 'v', "vmfssrc1", true, "Source directory for second file system passthrough (shared folder)" ), - VM_FSTGT1 ( 'w', "vmfstgt1", true, "Target directory for second file system passthrough (shared folder)" ); + HELP ( 'h', "help", 0, "" ), + DEBUG ( 'b', "debug", 1, "Enable or disable debug mode" ), + VM_CFGINP ( 'i', "vmcfginp", 1, "File name of an existing and filtered Libvirt domain XML configuration file" ), + VM_CFGOUT ( 'o', "vmcfgout", 1, "File name to output a finalized Libvirt domain XML configuration file" ), + VM_NAME ( 'n', "vmname", 1, "Name for the virtual machine" ), + VM_UUID ( 'u', "vmuuid", 1, "UUID for the virtual machine" ), + VM_DSPLNAME ( 'd', "vmdsplname", 1, "Display name for the virtual machine" ), + VM_OS ( 's', "vmos", 1, "Operating system running in the virtual machine" ), + VM_NCPUS ( 'c', "vmncpus", 1, "Number of virtual CPUs for the virtual machine" ), + VM_MEM ( 'm', "vmmem", 1, "Amount of memory for the virtual machine" ), + VM_HDD0 ( 'r', "vmhdd0", 1, "Disk image for the first HDD device" ), + VM_FLOPPY0 ( 'f', "vmfloppy0", 1, "Disk image for the first floppy drive" ), + VM_FLOPPY1 ( 'g', "vmfloppy1", 1, "Disk image for the second floppy drive" ), + VM_CDROM0 ( 'k', "vmcdrom0", 1, "Disk image for the first CDROM drive" ), + VM_CDROM1 ( 'l', "vmcdrom1", 1, "Disk image for the second CDROM drive" ), + VM_PARALLEL0( 'p', "vmparallel0", 1, "Device for the first parallel port interface" ), + VM_SERIAL0 ( 'q', "vmserial0", 1, "Device for the first serial port interface" ), + VM_MAC0 ( 'a', "vmmac0", 1, "MAC address for the first network interface" ), + VM_FSSRC0 ( 't', "vmfssrc0", 1, "Source directory for first file system passthrough (shared folder)" ), + VM_FSTGT0 ( 'e', "vmfstgt0", 1, "Target directory for first file system passthrough (shared folder)" ), + VM_FSSRC1 ( 'v', "vmfssrc1", 1, "Source directory for second file system passthrough (shared folder)" ), + VM_FSTGT1 ( 'w', "vmfstgt1", 1, "Target directory for second file system passthrough (shared folder)" ), + VM_NVGPUIDS0( 'y', "vmnvgpuids0", 2, "PCI device description and address for passthrough of the first Nvidia GPU. " + + "The argument follow the pattern: " + + "\":,::.\"" ); // @formatter:on /** @@ -389,9 +443,9 @@ public class CommandLineArgs private final String longOption; /** - * Stores the presence of an argument for the command line option. + * Stores the number of arguments for the command line option. */ - private final boolean hasArgument; + private final int numArguments; /** * Stores the textual description of the command line option. @@ -403,14 +457,14 @@ public class CommandLineArgs * * @param shortOption {@link Character} for the short command line option. * @param longOption {@link String} for the long command line option. - * @param hasArgument presence of an argument for the command line option. + * @param numArguments number of arguments for the command line option. * @param description textual description of the command line option. */ - CmdLnOption( char shortOption, String longOption, boolean hasArgument, String description ) + CmdLnOption( char shortOption, String longOption, int numArguments, String description ) { this.shortOption = shortOption; this.longOption = longOption; - this.hasArgument = hasArgument; + this.numArguments = numArguments; this.description = description; } @@ -435,13 +489,13 @@ public class CommandLineArgs } /** - * Returns the presence of an argument for the command line option. + * Returns the number of arguments for the command line option. * - * @return presence of an argument for the command line option. + * @return number of arguments for the command line option. */ - public boolean hasArgument() + public int getNumArguments() { - return this.hasArgument; + return this.numArguments; } /** diff --git a/core/modules/qemu/runvirt-plugin-qemu/src/main/java/org/openslx/runvirt/plugin/qemu/configuration/TransformationSpecificQemuArchitecture.java b/core/modules/qemu/runvirt-plugin-qemu/src/main/java/org/openslx/runvirt/plugin/qemu/configuration/TransformationSpecificQemuArchitecture.java index f507237d..a51c829d 100644 --- a/core/modules/qemu/runvirt-plugin-qemu/src/main/java/org/openslx/runvirt/plugin/qemu/configuration/TransformationSpecificQemuArchitecture.java +++ b/core/modules/qemu/runvirt-plugin-qemu/src/main/java/org/openslx/runvirt/plugin/qemu/configuration/TransformationSpecificQemuArchitecture.java @@ -29,14 +29,6 @@ public class TransformationSpecificQemuArchitecture */ private static final String NAME = "QEMU Architecture [CPU architecture, machine type, ...]"; - /** - * Capabilities of the Libvirt/QEMU hypervisor. - * - * @implNote This field is used as an instance of a singelton. Please always use - * {@link #getCapabilities()} to retrieve the {@code capabilities} instance. - */ - private Capabilities capabilities = null; - /** * Creates a new architecture transformation for Libvirt/QEMU virtualization configurations. * @@ -70,18 +62,17 @@ public class TransformationSpecificQemuArchitecture */ protected Capabilities getCapabilities() throws TransformationException { - // retrieve capabilities from QEMU hypervisor only once - if ( this.capabilities == null ) { - try { - this.capabilities = this.getVirtualizer().getCapabilites(); - } catch ( LibvirtHypervisorException e ) { - final String errorMsg = new String( - "Failed to get host capabilities from QEMU virtualizer: " + e.getLocalizedMessage() ); - throw new TransformationException( errorMsg ); - } + final Capabilities capabilities; + + try { + capabilities = this.getVirtualizer().getCapabilites(); + } catch ( LibvirtHypervisorException e ) { + final String errorMsg = new String( + "Failed to retrieve host capabilities from QEMU virtualizer: " + e.getLocalizedMessage() ); + throw new TransformationException( errorMsg ); } - return this.capabilities; + return capabilities; } /** diff --git a/core/modules/qemu/runvirt-plugin-qemu/src/main/java/org/openslx/runvirt/plugin/qemu/configuration/TransformationSpecificQemuGpuPassthroughNvidia.java b/core/modules/qemu/runvirt-plugin-qemu/src/main/java/org/openslx/runvirt/plugin/qemu/configuration/TransformationSpecificQemuGpuPassthroughNvidia.java index c41f989c..a22bf027 100644 --- a/core/modules/qemu/runvirt-plugin-qemu/src/main/java/org/openslx/runvirt/plugin/qemu/configuration/TransformationSpecificQemuGpuPassthroughNvidia.java +++ b/core/modules/qemu/runvirt-plugin-qemu/src/main/java/org/openslx/runvirt/plugin/qemu/configuration/TransformationSpecificQemuGpuPassthroughNvidia.java @@ -1,8 +1,19 @@ package org.openslx.runvirt.plugin.qemu.configuration; +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.List; + +import org.openslx.libvirt.capabilities.Capabilities; import org.openslx.libvirt.domain.Domain; +import org.openslx.libvirt.domain.device.HostdevPci; +import org.openslx.libvirt.domain.device.HostdevPciDeviceAddress; +import org.openslx.libvirt.domain.device.HostdevPciDeviceDescription; +import org.openslx.libvirt.domain.device.Shmem; +import org.openslx.libvirt.domain.device.Video; import org.openslx.runvirt.plugin.qemu.cmdln.CommandLineArgs; import org.openslx.runvirt.plugin.qemu.virtualization.LibvirtHypervisorQemu; +import org.openslx.runvirt.virtualization.LibvirtHypervisorException; import org.openslx.virtualization.configuration.transformation.TransformationException; import org.openslx.virtualization.configuration.transformation.TransformationSpecific; @@ -20,6 +31,31 @@ public class TransformationSpecificQemuGpuPassthroughNvidia */ private static final String NAME = "QEMU GPU passthrough [Nvidia]"; + /** + * Vendor identifier of PCI devices from Nvidia. + */ + private static final int NVIDIA_PCI_VENDOR_ID = 0x10de; + + /** + * Vendor identifier of the Hyper-V enlightenment for hypervisor shadowing. + */ + public static final String HYPERV_VENDOR_ID = "62776c706277"; + + /** + * Maximum width in pixel of the GPU passthrough rendered display. + */ + private static final long MAX_DISPLAY_WIDTH = 2560; + + /** + * Maximum height in pixel of the GPU passthrough rendered display. + */ + private static final long MAX_DISPLAY_HEIGHT = 1440; + + /** + * Reserved memory for framebuffer meta data of the Looking Glass shared memory device in MiB. + */ + private static final long RESERVED_MEMORY_FRAMEBUFFER = 10; + /** * Creates a new Nvidia GPU passthrough transformation for Libvirt/QEMU virtualization * configurations. @@ -31,6 +67,63 @@ public class TransformationSpecificQemuGpuPassthroughNvidia super( TransformationSpecificQemuGpuPassthroughNvidia.NAME, hypervisor ); } + /** + * Validates a PCI device description and address of a PCI device from a Nvidia GPU and parses + * the validated PCI device addresses. + * + * @param pciIds textual PCI device description and address to be validated. + * + * @return list of validated and parsed PCI device addresses for a NVIDIA GPU passthrough. + * + * @throws TransformationException validation of PCI device description and address failed. + */ + private static List validateParseNvidiaPciIds( List pciIds ) + throws TransformationException + { + final List parsedPciAddresses = new ArrayList(); + + if ( pciIds != null && pciIds.size() > 0 ) { + // abort if arguments do not follow the pattern: + // + // [0]: : + // [1]: ::. + // [2]: : + // [3]: ::. + // ... + // + if ( pciIds.size() % 2 != 0 ) { + throw new TransformationException( + "Arguments of PCI IDs are not follow the pattern for a GPU passthrough!" ); + } + + // parse PCI device description and PCI device address + for ( int i = 0; i < pciIds.size(); i += 2 ) { + // parse vendor and device ID + HostdevPciDeviceDescription deviceDescription = null; + try { + deviceDescription = HostdevPciDeviceDescription.valueOf( pciIds.get( i ) ); + } catch ( IllegalArgumentException e ) { + throw new TransformationException( "Invalid vendor or device ID of the PCI device description!" ); + } + + // validate vendor ID + final int vendorId = deviceDescription.getVendorId(); + if ( TransformationSpecificQemuGpuPassthroughNvidia.NVIDIA_PCI_VENDOR_ID != vendorId ) { + final String errorMsg = "Vendor ID '" + vendorId + "' of the PCI device is not from Nvidia!"; + throw new TransformationException( errorMsg ); + } + + // parse PCI domain, PCI bus, PCI device and PCI function + final HostdevPciDeviceAddress parsedPciAddress = HostdevPciDeviceAddress.valueOf( pciIds.get( i + 1 ) ); + if ( parsedPciAddress != null ) { + parsedPciAddresses.add( parsedPciAddress ); + } + } + } + + return parsedPciAddresses; + } + /** * Validates a virtualization configuration and input arguments for this transformation. * @@ -43,6 +136,55 @@ public class TransformationSpecificQemuGpuPassthroughNvidia if ( config == null || args == null ) { throw new TransformationException( "Virtualization configuration or input arguments are missing!" ); } + + TransformationSpecificQemuGpuPassthroughNvidia.validateParseNvidiaPciIds( args.getVmNvGpuIds0() ); + } + + /** + * Queries and returns the capabilities of the Libvirt/QEMU hypervisor. + * + * @return capabilities of the Libvirt/QEMU hypervisor. + * @throws TransformationException failed to query and return the capabilities of the + * Libvirt/QEMU hypervisor. + */ + protected Capabilities getCapabilities() throws TransformationException + { + Capabilities capabilities = null; + + try { + capabilities = this.getVirtualizer().getCapabilites(); + } catch ( LibvirtHypervisorException e ) { + final String errorMsg = new String( + "Failed to retrieve host capabilities from QEMU virtualizer: " + e.getLocalizedMessage() ); + throw new TransformationException( errorMsg ); + } + + return capabilities; + } + + private static BigInteger roundToNearestPowerOf2( BigInteger value ) + { + BigInteger k = BigInteger.valueOf( 1 ); + + while ( k.compareTo( value ) == -1 ) { + k = k.multiply( BigInteger.valueOf( 2 ) ); + } + + return k; + } + + /** + * Calculates the framebuffer memory size for the Looking Glass shared memory device. + * + * @return framebuffer memory size in bytes for the Looking Glass shared memory device. + */ + private static BigInteger calculateFramebufferSize() + { + final long totalBytesFramebuffer = MAX_DISPLAY_WIDTH * MAX_DISPLAY_HEIGHT * 4 * 2; + final long totalBytesReserved = RESERVED_MEMORY_FRAMEBUFFER * 1048576; + + // round sum of total memory in bytes to nearest power of two + return roundToNearestPowerOf2( BigInteger.valueOf( totalBytesFramebuffer + totalBytesReserved ) ); } @Override @@ -51,9 +193,40 @@ public class TransformationSpecificQemuGpuPassthroughNvidia // validate configuration and input arguments this.validateInputs( config, args ); - // check if IOMMU support is available on the host + // check if passthrough of Nvidia GPU takes place + if ( args.isNvidiaGpuPassthroughEnabled() ) { + // validate submitted PCI IDs + final List pciDeviceAddresses = TransformationSpecificQemuGpuPassthroughNvidia + .validateParseNvidiaPciIds( args.getVmNvGpuIds0() ); + + // check if IOMMU support is available on the host + if ( !this.getCapabilities().hasHostIommuSupport() ) { + final String errorMsg = "IOMMU support is not available on the hypervisor but required for GPU passthrough!"; + throw new TransformationException( errorMsg ); + } + + // passthrough PCI devices of the GPU + for ( final HostdevPciDeviceAddress pciDeviceAddress : pciDeviceAddresses ) { + final HostdevPci pciDevice = config.addHostdevPciDevice(); + pciDevice.setManaged( true ); + pciDevice.setSource( pciDeviceAddress ); + } - // TODO: implement Nvidia hypervisor shadowing - // call this filter at the end, since -> override of software graphics to 'none' necessary + // add shared memory device for Looking Glass + final Shmem shmemDevice = config.addShmemDevice(); + shmemDevice.setName( "looking-glass" ); + shmemDevice.setModel( Shmem.Model.IVSHMEM_PLAIN ); + shmemDevice.setSize( TransformationSpecificQemuGpuPassthroughNvidia.calculateFramebufferSize() ); + + // enable hypervisor shadowing to avoid error code 43 of Nvidia drivers in virtual machines + config.setFeatureHypervVendorIdValue( TransformationSpecificQemuGpuPassthroughNvidia.HYPERV_VENDOR_ID ); + config.setFeatureHypervVendorIdState( true ); + config.setFeatureKvmHiddenState( true ); + + // disable all software video devices by disable them + for ( Video videoDevice : config.getVideoDevices() ) { + videoDevice.disable(); + } + } } } diff --git a/core/modules/qemu/runvirt-plugin-qemu/src/main/java/org/openslx/runvirt/viewer/ViewerLookingGlassClient.java b/core/modules/qemu/runvirt-plugin-qemu/src/main/java/org/openslx/runvirt/viewer/ViewerLookingGlassClient.java new file mode 100644 index 00000000..cea9ccd8 --- /dev/null +++ b/core/modules/qemu/runvirt-plugin-qemu/src/main/java/org/openslx/runvirt/viewer/ViewerLookingGlassClient.java @@ -0,0 +1,105 @@ +package org.openslx.runvirt.viewer; + +import org.openslx.runvirt.virtualization.LibvirtHypervisor; +import org.openslx.runvirt.virtualization.LibvirtVirtualMachine; +import org.openslx.virtualization.Version; + +/** + * Looking Glass Client to view the exposed framebuffer (through a shared memory) of a virtual + * machine running the Looking Glass Host application. + * + * @author Manuel Bentele + * @version 1.0 + */ +public class ViewerLookingGlassClient extends Viewer +{ + /** + * Name of the Looking Glass Client program. + */ + private final static String NAME = "looking-glass-client"; + + /** + * Maximum number of supported displays by the Looking Glass Client. + */ + private final static int NUM_SUPPORTED_DISPLAYS = 1; + + /** + * File name of the shared memory file to receive display content from the Looking Glass Host. + */ + private final static String SHARED_MEMORY_FILENAME = "/dev/shm/looking-glass"; + + /** + * State whether showing debug information during virtual machine rendering or not. + */ + private final boolean debug; + + /** + * Creates a new Looking Glass Client for a Libvirt virtual machine running on a Libvirt + * hypervisor. + * + * @param machine virtual machine to display. + * @param hypervisor remote (hypervisor) endpoint for the viewer to connect to. + */ + public ViewerLookingGlassClient( LibvirtVirtualMachine machine, LibvirtHypervisor hypervisor ) + { + this( machine, hypervisor, false ); + } + + /** + * Creates a new Looking Glass Client for a Libvirt virtual machine running on a Libvirt + * hypervisor. + * + * @param machine virtual machine to display. + * @param hypervisor remote (hypervisor) endpoint for the viewer to connect to. + * @param debug state whether showing debug information during virtual machine rendering or not. + */ + public ViewerLookingGlassClient( LibvirtVirtualMachine machine, LibvirtHypervisor hypervisor, boolean debug ) + { + super( ViewerLookingGlassClient.NAME, ViewerLookingGlassClient.NUM_SUPPORTED_DISPLAYS, machine, hypervisor ); + + this.debug = debug; + } + + /** + * Returns the state whether showing debug information during virtual machine rendering or not. + * + * @return state whether showing debug information during virtual machine rendering or not. + */ + public boolean isDebugEnabled() + { + return this.debug; + } + + @Override + public Version getVersion() throws ViewerException + { + return null; + } + + @Override + public void render() throws ViewerException + { + // execute viewer process with arguments: + // in non-debug mode: + // "looking-glass-client app:shmFile= win:fullScreen=yes spice:enable=yes win:alerts=no" + // in debug mode: + // "looking-glass-client app:shmFile= win:fullScreen=yes spice:enable=yes win:alerts=yes win:showFPS=yes" + final String[] viewerParameters; + if ( this.isDebugEnabled() ) { + viewerParameters = new String[] { + "app:shmFile=" + ViewerLookingGlassClient.SHARED_MEMORY_FILENAME, + "win:fullScreen=yes", + "spice:enable=yes", + "win:alerts=no" }; + } else { + viewerParameters = new String[] { + "app:shmFile=" + ViewerLookingGlassClient.SHARED_MEMORY_FILENAME, + "win:fullScreen=yes", + "spice:enable=yes", + "win:alerts=yes", + "win:showFPS=yes" }; + } + + ViewerUtils.executeViewer( ViewerLookingGlassClient.NAME, viewerParameters ); + } +} diff --git a/core/modules/qemu/runvirt-plugin-qemu/src/test/java/org/openslx/runvirt/plugin/qemu/AppTest.java b/core/modules/qemu/runvirt-plugin-qemu/src/test/java/org/openslx/runvirt/plugin/qemu/AppTest.java index 1db1525b..d0eef82a 100644 --- a/core/modules/qemu/runvirt-plugin-qemu/src/test/java/org/openslx/runvirt/plugin/qemu/AppTest.java +++ b/core/modules/qemu/runvirt-plugin-qemu/src/test/java/org/openslx/runvirt/plugin/qemu/AppTest.java @@ -63,7 +63,7 @@ public class AppTest assertTrue( shortHelpOptionCorrectOutput.contains( App.APP_DESC ) ); // test that no error was logged and output is available - assertEquals( 2160, shortHelpOptionCorrectOutput.length() ); + assertEquals( 2503, shortHelpOptionCorrectOutput.length() ); assertEquals( 0, shortHelpOptionCorrectErrOutput.length() ); } @@ -91,7 +91,7 @@ public class AppTest assertTrue( longHelpOptionCorrectOutput.contains( App.APP_DESC ) ); // test that no error was logged and output is available - assertEquals( 2160, longHelpOptionCorrectOutput.length() ); + assertEquals( 2503, longHelpOptionCorrectOutput.length() ); assertEquals( 0, longHelpOptionCorrectErrOutput.length() ); } @@ -119,7 +119,7 @@ public class AppTest assertTrue( shortHelpOptionIncorrectOutput.contains( App.APP_DESC ) ); // test that error was logged and output is available - assertEquals( 2160, shortHelpOptionIncorrectOutput.length() ); + assertEquals( 2503, shortHelpOptionIncorrectOutput.length() ); assertEquals( 0, shortHelpOptionIncorrectErrOutput.length() ); } @@ -147,7 +147,7 @@ public class AppTest assertTrue( longHelpOptionIncorrectOutput.contains( App.APP_DESC ) ); // test that error was logged and output is available - assertEquals( 2160, longHelpOptionIncorrectOutput.length() ); + assertEquals( 2503, longHelpOptionIncorrectOutput.length() ); assertEquals( 0, longHelpOptionIncorrectErrOutput.length() ); } diff --git a/core/modules/qemu/runvirt-plugin-qemu/src/test/java/org/openslx/runvirt/plugin/qemu/cmdln/CommandLineArgsTest.java b/core/modules/qemu/runvirt-plugin-qemu/src/test/java/org/openslx/runvirt/plugin/qemu/cmdln/CommandLineArgsTest.java index 972f5e4b..77522bd6 100644 --- a/core/modules/qemu/runvirt-plugin-qemu/src/test/java/org/openslx/runvirt/plugin/qemu/cmdln/CommandLineArgsTest.java +++ b/core/modules/qemu/runvirt-plugin-qemu/src/test/java/org/openslx/runvirt/plugin/qemu/cmdln/CommandLineArgsTest.java @@ -6,6 +6,7 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; import java.io.File; +import java.util.List; import org.junit.jupiter.api.DisplayName; import org.junit.jupiter.api.Test; @@ -28,6 +29,8 @@ public class CommandLineArgsTest private static final String CMDLN_TEST_PARPORT = "/dev/parport0"; private static final String CMDLN_TEST_SERPORT = "/dev/ttyS0"; private static final String CMDLN_TEST_MAC = "02:42:8e:77:1b:e6"; + private static final String CMDLN_TEST_NVGPU_DESC = "10de:0ff9"; + private static final String CMDLN_TEST_NVGPU_ADDR = "0000:00:01.0"; // @formatter:on @Test @@ -695,4 +698,55 @@ public class CommandLineArgsTest assertEquals( CMDLN_TEST_NAME, cmdLn.getVmFsTgt1() ); } + + @Test + @DisplayName( "Test the parsing of NVIDIA PCI IDs command line option for the first GPU passthrough (short version)" ) + public void testCmdlnOptionVmNvGpuIds0Short() throws CommandLineArgsException + { + final String[] args = { + CMDLN_PREFIX_OPTION_SHORT + CmdLnOption.VM_NVGPUIDS0.getShortOption(), + CMDLN_TEST_NVGPU_DESC, CMDLN_TEST_NVGPU_ADDR + }; + + CommandLineArgs cmdLn = new CommandLineArgs( args ); + + final List nvidiaGpuIds = cmdLn.getVmNvGpuIds0(); + assertEquals( 2, nvidiaGpuIds.size() ); + assertEquals( CMDLN_TEST_NVGPU_DESC, nvidiaGpuIds.get( 0 ) ); + assertEquals( CMDLN_TEST_NVGPU_ADDR, nvidiaGpuIds.get( 1 ) ); + } + + @Test + @DisplayName( "Test the parsing of NVIDIA PCI IDs command line option for the first GPU passthrough (long version)" ) + public void testCmdlnOptionVmNvGpuIds0Long() throws CommandLineArgsException + { + final String[] args = { + CMDLN_PREFIX_OPTION_LONG + CmdLnOption.VM_NVGPUIDS0.getLongOption(), + CMDLN_TEST_NVGPU_DESC, CMDLN_TEST_NVGPU_ADDR + }; + + CommandLineArgs cmdLn = new CommandLineArgs( args ); + + final List nvidiaGpuIds = cmdLn.getVmNvGpuIds0(); + assertEquals( 2, nvidiaGpuIds.size() ); + assertEquals( CMDLN_TEST_NVGPU_DESC, nvidiaGpuIds.get( 0 ) ); + assertEquals( CMDLN_TEST_NVGPU_ADDR, nvidiaGpuIds.get( 1 ) ); + } + + @Test + @DisplayName( "Test whether a NVIDIA GPU passthrough is enabled" ) + public void testIsNvidiaGpuPassthroughEnabled() throws CommandLineArgsException + { + final String[] args1 = { + CMDLN_PREFIX_OPTION_LONG + CmdLnOption.VM_NVGPUIDS0.getLongOption(), + CMDLN_TEST_NVGPU_DESC, CMDLN_TEST_NVGPU_ADDR + }; + final String[] args2 = {}; + + CommandLineArgs cmdLn1 = new CommandLineArgs( args1 ); + CommandLineArgs cmdLn2 = new CommandLineArgs( args2 ); + + assertTrue( cmdLn1.isNvidiaGpuPassthroughEnabled() ); + assertFalse( cmdLn2.isNvidiaGpuPassthroughEnabled() ); + } } diff --git a/core/modules/qemu/runvirt-plugin-qemu/src/test/java/org/openslx/runvirt/plugin/qemu/configuration/TransformationSpecificQemuGpuPassthroughNvidiaTest.java b/core/modules/qemu/runvirt-plugin-qemu/src/test/java/org/openslx/runvirt/plugin/qemu/configuration/TransformationSpecificQemuGpuPassthroughNvidiaTest.java new file mode 100644 index 00000000..3a9624b3 --- /dev/null +++ b/core/modules/qemu/runvirt-plugin-qemu/src/test/java/org/openslx/runvirt/plugin/qemu/configuration/TransformationSpecificQemuGpuPassthroughNvidiaTest.java @@ -0,0 +1,129 @@ +package org.openslx.runvirt.plugin.qemu.configuration; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +import java.io.InputStream; +import java.math.BigInteger; +import java.util.List; + +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.openslx.libvirt.capabilities.Capabilities; +import org.openslx.libvirt.domain.Domain; +import org.openslx.libvirt.domain.device.HostdevPci; +import org.openslx.libvirt.domain.device.HostdevPciDeviceAddress; +import org.openslx.libvirt.domain.device.Shmem; +import org.openslx.libvirt.domain.device.Video; +import org.openslx.libvirt.xml.LibvirtXmlDocumentException; +import org.openslx.libvirt.xml.LibvirtXmlSerializationException; +import org.openslx.libvirt.xml.LibvirtXmlTestResources; +import org.openslx.libvirt.xml.LibvirtXmlValidationException; +import org.openslx.runvirt.plugin.qemu.cmdln.CommandLineArgs; +import org.openslx.virtualization.configuration.transformation.TransformationException; + +class TransformationSpecificQemuGpuPassthroughNvidiaStub extends TransformationSpecificQemuGpuPassthroughNvidia +{ + final String capabilityFileName; + + public TransformationSpecificQemuGpuPassthroughNvidiaStub( String capabilityFileName ) + { + super( null ); + + this.capabilityFileName = capabilityFileName; + } + + @Override + protected Capabilities getCapabilities() throws TransformationException + { + final InputStream capabilityContent = LibvirtXmlTestResources.getLibvirtXmlStream( this.capabilityFileName ); + Capabilities capabilites = null; + + try { + capabilites = new Capabilities( capabilityContent ); + } catch ( LibvirtXmlDocumentException | LibvirtXmlSerializationException | LibvirtXmlValidationException e ) { + fail( "Could not create stub for getCapabilities(): " + e.getLocalizedMessage() ); + } + + return capabilites; + } +} + +public class TransformationSpecificQemuGpuPassthroughNvidiaTest +{ + @Test + @DisplayName( "Test transformation of VM GPU passthrough configuration if NVIDIA GPU passthrouh is required" ) + public void testTransformationSpecificQemuGpuPassthroughNvidia() throws TransformationException + { + final TransformationSpecificQemuGpuPassthroughNvidiaStub transformation; + transformation = new TransformationSpecificQemuGpuPassthroughNvidiaStub( "qemu-kvm_capabilities_default.xml" ); + final Domain config = TransformationTestUtils.getDefaultDomain(); + final CommandLineArgs args = TransformationTestUtils.getDefaultCmdLnArgs(); + + transformation.transform( config, args ); + + final List pciDevices = config.getHostdevPciDevices(); + assertNotNull( pciDevices ); + assertEquals( 1, pciDevices.size() ); + + final HostdevPci pciDevice = pciDevices.get( 0 ); + assertTrue( pciDevice.isManaged() ); + assertEquals( HostdevPciDeviceAddress.valueOf( TransformationTestUtils.DEFAULT_VM_GPU0_ADDR ), + pciDevice.getSource() ); + + final List shmemDevices = config.getShmemDevices(); + assertNotNull( shmemDevices ); + assertEquals( 1, shmemDevices.size() ); + + final Shmem shmemDevice = shmemDevices.get( 0 ); + assertEquals( "looking-glass", shmemDevice.getName() ); + assertEquals( Shmem.Model.IVSHMEM_PLAIN, shmemDevice.getModel() ); + assertEquals( BigInteger.valueOf( 67108864 ).toString(), shmemDevice.getSize().toString() ); + + assertEquals( TransformationSpecificQemuGpuPassthroughNvidia.HYPERV_VENDOR_ID, + config.getFeatureHypervVendorIdValue() ); + assertTrue( config.isFeatureHypervVendorIdStateOn() ); + assertTrue( config.isFeatureKvmHiddenStateOn() ); + + final List