From 7937083c2df8f1547ce18a9cdd58eaa9c203e073 Mon Sep 17 00:00:00 2001 From: Simon Rettberg Date: Fri, 2 Jul 2021 11:20:12 +0200 Subject: [vmware] Less strict filtering Suite <-> dmsd, redo filtering logic Get rid of the old flags for different filtering targets. This was never implemented properly anyways and was confusing to use. Instead, add explicit on-demand filtering to to according transformation methods. --- .../VirtualizationConfigurationVmware.java | 125 +++++++++++++-------- ...irtualizationConfigurationVmwareFileFormat.java | 43 ++----- ...gurationLogicDozModServerToStatelessClient.java | 14 +-- ...urationLogicDozModClientToDozModServerTest.java | 1 + ...urationLogicDozModServerToDozModClientTest.java | 1 + ...tionLogicDozModServerToStatelessClientTest.java | 1 + ...re-player_default-ubuntu_transform-editable.vmx | 15 +++ ...are-player_default-ubuntu_transform-privacy.vmx | 19 ++++ 8 files changed, 134 insertions(+), 85 deletions(-) diff --git a/src/main/java/org/openslx/virtualization/configuration/VirtualizationConfigurationVmware.java b/src/main/java/org/openslx/virtualization/configuration/VirtualizationConfigurationVmware.java index 0cbb50c..2c2013a 100644 --- a/src/main/java/org/openslx/virtualization/configuration/VirtualizationConfigurationVmware.java +++ b/src/main/java/org/openslx/virtualization/configuration/VirtualizationConfigurationVmware.java @@ -4,6 +4,7 @@ import java.io.File; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.util.HashMap; +import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Map.Entry; @@ -72,27 +73,31 @@ public class VirtualizationConfigurationVmware extends private static final Logger LOGGER = Logger.getLogger( VirtualizationConfigurationVmware.class ); - private static final Pattern hddKey = Pattern.compile( "^(ide\\d|scsi\\d|sata\\d|nvme\\d):?(\\d)?\\.(.*)", + private static final Pattern HDD_PATTERN = Pattern.compile( "^(ide\\d|scsi\\d|sata\\d|nvme\\d):?(\\d?)\\.(.*)", Pattern.CASE_INSENSITIVE ); - // Lowercase list of allowed settings for upload (as regex) - private static final Pattern[] whitelist; + /** Lowercase regex of allowed settings for stateless execution */ + private static final Pattern STATELESS_WHITELIST_PATTERN; + + /** Lowercase regex of forbidden settings when uploading (privacy concerns) */ + private static final Pattern PRIVACY_BLACKLIST_PATTERN; private final VirtualizationConfigurationVmwareFileFormat config; // Init static members static { - String[] list = { "^guestos", "^uuid\\.bios", "^config\\.version", "^ehci[.:]", "^mks\\.enable3d", + // LOWERCASE - Client execution whitelist + String[] list1 = { "^guestos", "^uuid\\.bios", "^config\\.version", "^ehci[.:]", "^mks\\.enable3d", "^virtualhw\\.", "^sound[.:]", "\\.pcislotnumber$", "^pcibridge", "\\.virtualdev$", "^tools\\.syncTime$", "^time\\.synchronize", "^bios\\.bootDelay", "^rtc\\.", "^xhci[.:]", "^usb_xhci[.:]", "\\.deviceType$", "\\.port$", "\\.parent$", "^usb[.:]", "^firmware", "^hpet", "^vm\\.genid" }; - whitelist = new Pattern[ list.length ]; - for ( int i = 0; i < list.length; ++i ) { - whitelist[i] = Pattern.compile( list[i].toLowerCase() ); - } + STATELESS_WHITELIST_PATTERN = Pattern.compile( String.join( "|", list1 ).toLowerCase(), Pattern.CASE_INSENSITIVE ); + // LOWERCASE - Upload privacy filter + String[] list2 = { "^displayname$", "^extendedconfigfile$", "^gui\\.", "^nvram$", "^memsize$" }; + PRIVACY_BLACKLIST_PATTERN = Pattern.compile( String.join( "|", list2 ).toLowerCase(), Pattern.CASE_INSENSITIVE ); } public static enum EthernetType @@ -107,8 +112,6 @@ public class VirtualizationConfigurationVmware extends } } - private final Map disks = new HashMap<>(); - public VirtualizationConfigurationVmware( List osList, File file ) throws IOException, VirtualizationConfigurationException { @@ -127,8 +130,9 @@ public class VirtualizationConfigurationVmware extends private void init() { + Map disks = new HashMap<>(); for ( Entry entry : config.entrySet() ) { - handleLoadEntry( entry ); + handleLoadEntry( entry, disks ); } // Fix accidentally filtered USB config if we see EHCI is present if ( isSetAndTrue( "ehci.present" ) && !isSetAndTrue( "usb.present" ) ) { @@ -136,6 +140,12 @@ public class VirtualizationConfigurationVmware extends } // if we find this tag, we already went through the hdd's - so we're done. if ( config.get( "#SLX_HDD_BUS" ) != null ) { + try { + hdds.add( new HardDisk( config.get( "#SLX_HDD_CHIP" ), + DriveBusType.valueOf( config.get( "#SLX_HDD_BUS" ) ), "empty" ) ); + } catch ( Exception e ) { + LOGGER.debug( "Error adding HDD object when parsing #SLX_HDD_BUS. Meta-data will be incorrect.", e ); + } return; } // Now find the HDDs and add to list @@ -145,6 +155,7 @@ public class VirtualizationConfigurationVmware extends if ( !controller.present ) continue; for ( Entry dEntry : controller.devices.entrySet() ) { + String deviceId = dEntry.getKey(); Device device = dEntry.getValue(); if ( !device.present ) continue; // Not present @@ -161,6 +172,8 @@ public class VirtualizationConfigurationVmware extends bus = DriveBusType.NVME; } hdds.add( new HardDisk( controller.virtualDev, bus, device.filename ) ); + // Remove original entries from VMX + removeEntriesStartingWith( controllerType + ":" + deviceId + "." ); } } // TODO check if this machine is in a paused/suspended state @@ -176,9 +189,19 @@ public class VirtualizationConfigurationVmware extends } } + private void removeEntriesStartingWith( String start ) + { + for ( Iterator> it = config.entrySet().iterator(); it.hasNext(); ) { + Entry entry = it.next(); + if ( entry.getKey().startsWith( start ) ) { + it.remove(); + } + } + } + private void addFiltered( String key, String value ) { - config.set( key, value ).filtered( true ); + config.set( key, value ); } private boolean isSetAndTrue( String key ) @@ -187,17 +210,9 @@ public class VirtualizationConfigurationVmware extends return value != null && value.equalsIgnoreCase( "true" ); } - private void handleLoadEntry( Entry entry ) + private void handleLoadEntry( Entry entry, Map disks ) { String lowerKey = entry.getKey().toLowerCase(); - // Cleaned vmx construction - for ( Pattern exp : whitelist ) { - if ( exp.matcher( lowerKey ).find() ) { - entry.getValue().filtered( true ); - break; - } - } - // // Dig Usable meta data String value = entry.getValue().getValue(); if ( lowerKey.equals( "guestos" ) ) { @@ -208,13 +223,13 @@ public class VirtualizationConfigurationVmware extends displayName = value; return; } - Matcher hdd = hddKey.matcher( entry.getKey() ); + Matcher hdd = HDD_PATTERN.matcher( entry.getKey() ); if ( hdd.find() ) { - handleHddEntry( hdd.group( 1 ).toLowerCase(), hdd.group( 2 ), hdd.group( 3 ), value ); + handleHddEntry( disks, hdd.group( 1 ).toLowerCase(), hdd.group( 2 ), hdd.group( 3 ), value ); } } - private void handleHddEntry( String controllerStr, String deviceStr, String property, String value ) + private void handleHddEntry( Map disks, String controllerStr, String deviceStr, String property, String value ) { Controller controller = disks.get( controllerStr ); if ( controller == null ) { @@ -265,16 +280,16 @@ public class VirtualizationConfigurationVmware extends return false; } - DriveBusType bus; - try { - bus = DriveBusType.valueOf( config.get( "#SLX_HDD_BUS" ) ); - } catch ( Exception e ) { - LOGGER.warn( "Unknown bus type: " + config.get( "#SLX_HDD_BUS" ) + ". Cannot add hdd config." ); + if ( hdds.isEmpty() ) { + LOGGER.warn( "No HDDs found in configuration" ); return false; } - String chipset = config.get( "#SLX_HDD_CHIP" ); + + HardDisk hdd = hdds.get( 0 ); + + String chipset = hdd.chipsetDriver; String prefix; - switch ( bus ) { + switch ( hdd.bus ) { case SATA: // Cannot happen?... use lsisas1068 prefix = "scsi0"; @@ -283,10 +298,10 @@ public class VirtualizationConfigurationVmware extends case IDE: case SCSI: case NVME: - prefix = bus.name().toLowerCase() + "0"; + prefix = hdd.bus.name().toLowerCase() + "0"; break; default: - LOGGER.warn( "Unknown HDD bus type: " + bus.toString() ); + LOGGER.warn( "Unknown HDD bus type: " + hdd.bus.toString() ); return false; } // Gen @@ -406,9 +421,38 @@ public class VirtualizationConfigurationVmware extends @Override public void transformNonPersistent() throws VirtualizationConfigurationException { + // Cleaned vmx construction + for ( Iterator> it = config.entrySet().iterator(); it.hasNext(); ) { + Entry elem = it.next(); + if ( !STATELESS_WHITELIST_PATTERN.matcher( elem.getKey() ).find() ) { + it.remove(); + } + } addFiltered( "suspend.disabled", "TRUE" ); } + @Override + public void transformEditable() throws VirtualizationConfigurationException + { + addFiltered( "gui.applyHostDisplayScalingToGuest", "FALSE" ); + } + + @Override + public void transformPrivacy() throws VirtualizationConfigurationException + { + for ( Iterator> it = config.entrySet().iterator(); it.hasNext(); ) { + Entry elem = it.next(); + String key = elem.getKey(); + String value = elem.getValue().getValue(); + if ( key.endsWith( ".fileName" ) && !value.startsWith( "-" ) + && ( value.contains( "." ) || value.contains( "/" ) || value.contains( "\\" ) ) ) { + it.remove(); + } else if ( PRIVACY_BLACKLIST_PATTERN.matcher( key ).find() ) { + it.remove(); + } + } + } + @Override public boolean addDisplayName( String name ) { @@ -434,7 +478,7 @@ public class VirtualizationConfigurationVmware extends public byte[] getConfigurationAsByteArray() { - return config.toString( false, false ).getBytes( StandardCharsets.UTF_8 ); + return config.toString().getBytes( StandardCharsets.UTF_8 ); } private static class Device @@ -463,12 +507,6 @@ public class VirtualizationConfigurationVmware extends } } - @Override - public void transformEditable() throws VirtualizationConfigurationException - { - addFiltered( "gui.applyHostDisplayScalingToGuest", "FALSE" ); - } - public String getValue( String key ) { return config.get( key ); @@ -607,15 +645,10 @@ public class VirtualizationConfigurationVmware extends @Override public boolean addCpuCoreCount( int numCores ) { - // TODO actually add the cpu core count to the machine description + addFiltered( "numvcpus", vmInteger( numCores ) ); return true; } - @Override - public void transformPrivacy() throws VirtualizationConfigurationException - { - } - public void registerVirtualHW() { soundCards.put( VirtualizationConfiguration.SoundCardType.NONE, new VmWareSoundCardMeta( false, null ) ); diff --git a/src/main/java/org/openslx/virtualization/configuration/VirtualizationConfigurationVmwareFileFormat.java b/src/main/java/org/openslx/virtualization/configuration/VirtualizationConfigurationVmwareFileFormat.java index 24df02f..231b2f2 100644 --- a/src/main/java/org/openslx/virtualization/configuration/VirtualizationConfigurationVmwareFileFormat.java +++ b/src/main/java/org/openslx/virtualization/configuration/VirtualizationConfigurationVmwareFileFormat.java @@ -95,11 +95,11 @@ public class VirtualizationConfigurationVmwareFileFormat while ( ( line = reader.readLine() ) != null ) { KeyValuePair entry = parse( line ); - if ( entry != null ) { + if ( entry != null ) { + // TODO: This is supposed to be case insensitive. + // Check if there are other consequences from lowercase entries in converted vmx files. if ( entry.key.equals( "virtualHW.version" ) || entry.key.equals( "ddb.virtualHWVersion" ) - // TODO: This is supposed to be case insensitive. - // Check if there are other consequences from lowercase entries in converted vmx files. - || entry.key.equals( "virtualhw.version" )) { + || entry.key.equals( "virtualhw.version" ) ) { isValid = true; } set( entry.key, unescape( entry.value ) ); @@ -218,51 +218,30 @@ public class VirtualizationConfigurationVmwareFileFormat return ce.value; } - public String toString( boolean filteredRequired, boolean generatedRequired ) + @Override + public String toString() { - set( ".encoding", "UTF-8" ).filtered( true ).generated( true ); + set( ".encoding", "UTF-8" ); StringBuilder sb = new StringBuilder( 300 ); for ( Entry entry : entries.entrySet() ) { ConfigEntry value = entry.getValue(); - if ( ( !filteredRequired || value.forFiltered ) && ( !generatedRequired || value.forGenerated ) ) { - sb.append( entry.getKey() ); - sb.append( " = \"" ); - sb.append( value.getEscaped() ); - sb.append( "\"\n" ); - } + sb.append( entry.getKey() ); + sb.append( " = \"" ); + sb.append( value.getEscaped() ); + sb.append( "\"\n" ); } return sb.toString(); } - @Override - public String toString() - { - return toString( false, false ); - } - public static class ConfigEntry { private String value; - private boolean forFiltered; - private boolean forGenerated; public ConfigEntry( String value ) { this.value = value; } - public ConfigEntry filtered( boolean set ) - { - this.forFiltered = set; - return this; - } - - public ConfigEntry generated( boolean set ) - { - this.forGenerated = set; - return this; - } - public String getEscaped() { String ret = value; diff --git a/src/main/java/org/openslx/virtualization/configuration/logic/ConfigurationLogicDozModServerToStatelessClient.java b/src/main/java/org/openslx/virtualization/configuration/logic/ConfigurationLogicDozModServerToStatelessClient.java index 30d690a..96d0b91 100644 --- a/src/main/java/org/openslx/virtualization/configuration/logic/ConfigurationLogicDozModServerToStatelessClient.java +++ b/src/main/java/org/openslx/virtualization/configuration/logic/ConfigurationLogicDozModServerToStatelessClient.java @@ -73,6 +73,13 @@ public class ConfigurationLogicDozModServerToStatelessClient // check if input parameters for a transformation are valid this.validateInputs( config, args ); + // apply settings to run virtualized system in a stateless manner + try { + config.transformNonPersistent(); + } catch ( VirtualizationConfigurationException e ) { + throw new TransformationException( e.getLocalizedMessage() ); + } + // set display name of lecture if ( !config.addDisplayName( args.getDisplayName() ) ) { throw new TransformationException( "Can not set display name in virtualization configuration!" ); @@ -98,12 +105,5 @@ public class ConfigurationLogicDozModServerToStatelessClient if ( !args.hasUsbAccess() ) { config.setMaxUsbSpeed( UsbSpeed.NONE ); } - - // apply settings to run virtualized system in a stateless manner - try { - config.transformNonPersistent(); - } catch ( VirtualizationConfigurationException e ) { - throw new TransformationException( e.getLocalizedMessage() ); - } } } diff --git a/src/test/java/org/openslx/virtualization/configuration/logic/ConfigurationLogicDozModClientToDozModServerTest.java b/src/test/java/org/openslx/virtualization/configuration/logic/ConfigurationLogicDozModClientToDozModServerTest.java index f078b5e..01408f0 100644 --- a/src/test/java/org/openslx/virtualization/configuration/logic/ConfigurationLogicDozModClientToDozModServerTest.java +++ b/src/test/java/org/openslx/virtualization/configuration/logic/ConfigurationLogicDozModClientToDozModServerTest.java @@ -79,6 +79,7 @@ public class ConfigurationLogicDozModClientToDozModServerTest final File expectedConfig = ConfigurationLogicTestResources.getVmwareVmxFile( expectedConfigFileName ); final VirtualizationConfiguration config; config = ConfigurationLogicTestUtils.newVirtualizationConfigurationInstance( inputConfig ); + assertTrue( config.getHdds().size() == 1 ); final ConfigurationLogicDozModClientToDozModServer logic = new ConfigurationLogicDozModClientToDozModServer(); logic.apply( config, new ConfigurationDataDozModClientToDozModServer() ); diff --git a/src/test/java/org/openslx/virtualization/configuration/logic/ConfigurationLogicDozModServerToDozModClientTest.java b/src/test/java/org/openslx/virtualization/configuration/logic/ConfigurationLogicDozModServerToDozModClientTest.java index 5ba5e18..844e5ec 100644 --- a/src/test/java/org/openslx/virtualization/configuration/logic/ConfigurationLogicDozModServerToDozModClientTest.java +++ b/src/test/java/org/openslx/virtualization/configuration/logic/ConfigurationLogicDozModServerToDozModClientTest.java @@ -95,6 +95,7 @@ public class ConfigurationLogicDozModServerToDozModClientTest final File expectedConfig = ConfigurationLogicTestResources.getVmwareVmxFile( expectedConfigFileName ); final VirtualizationConfiguration config; config = ConfigurationLogicTestUtils.newVirtualizationConfigurationInstance( inputConfig ); + assertTrue( config.getHdds().size() == 1 ); final ConfigurationLogicDozModServerToDozModClient logic = new ConfigurationLogicDozModServerToDozModClient(); logic.apply( config, ConfigurationLogicDozModServerToDozModClientTest.DEFAULT_CONFIG_DATA ); diff --git a/src/test/java/org/openslx/virtualization/configuration/logic/ConfigurationLogicDozModServerToStatelessClientTest.java b/src/test/java/org/openslx/virtualization/configuration/logic/ConfigurationLogicDozModServerToStatelessClientTest.java index 6c87526..47d8a45 100644 --- a/src/test/java/org/openslx/virtualization/configuration/logic/ConfigurationLogicDozModServerToStatelessClientTest.java +++ b/src/test/java/org/openslx/virtualization/configuration/logic/ConfigurationLogicDozModServerToStatelessClientTest.java @@ -89,6 +89,7 @@ public class ConfigurationLogicDozModServerToStatelessClientTest final File expectedConfig = ConfigurationLogicTestResources.getVmwareVmxFile( expectedConfigFileName ); final VirtualizationConfiguration config; config = ConfigurationLogicTestUtils.newVirtualizationConfigurationInstance( inputConfig ); + assertTrue( config.getHdds().size() == 1 ); final ConfigurationLogicDozModServerToStatelessClient logic = new ConfigurationLogicDozModServerToStatelessClient(); logic.apply( config, ConfigurationLogicDozModServerToStatelessClientTest.DEFAULT_CONFIG_DATA ); diff --git a/src/test/resources/vmware/vmx/vmware-player_default-ubuntu_transform-editable.vmx b/src/test/resources/vmware/vmx/vmware-player_default-ubuntu_transform-editable.vmx index 9d1ad5e..3972aa0 100644 --- a/src/test/resources/vmware/vmx/vmware-player_default-ubuntu_transform-editable.vmx +++ b/src/test/resources/vmware/vmx/vmware-player_default-ubuntu_transform-editable.vmx @@ -2,6 +2,7 @@ config.version = "8" displayName = "Test" ehci.present = "TRUE" +ethernet0.addressType = "generated" ethernet0.connectionType = "nat" ethernet0.present = "TRUE" ethernet0.virtualDev = "e1000" @@ -22,8 +23,10 @@ ide0:0.present = "TRUE" ide0:1.autodetect = "TRUE" ide0:1.deviceType = "cdrom-raw" ide0:1.present = "TRUE" +mem.hotadd = "TRUE" memsize = "1536" mks.enable3d = "TRUE" +numvcpus = "1" pciBridge0.present = "TRUE" pciBridge4.functions = "8" pciBridge4.present = "TRUE" @@ -37,18 +40,30 @@ pciBridge6.virtualDev = "pcieRootPort" pciBridge7.functions = "8" pciBridge7.present = "TRUE" pciBridge7.virtualDev = "pcieRootPort" +powerType.powerOff = "soft" +powerType.powerOn = "soft" +powerType.reset = "soft" +powerType.suspend = "soft" +sata0.present = "TRUE" sata0:1.deviceType = "cdrom-raw" +sata0:1.present = "TRUE" scsi0.present = "TRUE" scsi0.virtualDev = "lsilogic" scsi0:0.deviceType = "disk" scsi0:0.fileName = "image-default.vmdk" scsi0:0.present = "TRUE" +serial0.fileName = "thinprint" +serial0.fileType = "thinprint" +serial0.present = "TRUE" sound.autoDetect = "TRUE" sound.fileName = "-1" sound.present = "TRUE" +svga.graphicsMemoryKB = "8388608" tools.syncTime = "FALSE" usb.mangleUsb3Speed = "TRUE" usb.present = "TRUE" usb.vbluetooth.startConnected = "TRUE" +vcpu.hotadd = "TRUE" virtualHW.productCompatibility = "hosted" virtualHW.version = "18" +vmci0.present = "TRUE" diff --git a/src/test/resources/vmware/vmx/vmware-player_default-ubuntu_transform-privacy.vmx b/src/test/resources/vmware/vmx/vmware-player_default-ubuntu_transform-privacy.vmx index a08b044..9de07ce 100644 --- a/src/test/resources/vmware/vmx/vmware-player_default-ubuntu_transform-privacy.vmx +++ b/src/test/resources/vmware/vmx/vmware-player_default-ubuntu_transform-privacy.vmx @@ -3,10 +3,16 @@ .encoding = "UTF-8" config.version = "8" ehci.present = "TRUE" +ethernet0.addressType = "generated" +ethernet0.connectionType = "nat" +ethernet0.present = "TRUE" ethernet0.virtualDev = "e1000" +floppy0.present = "FALSE" guestOS = "ubuntu-64" hpet0.present = "TRUE" +mem.hotadd = "TRUE" mks.enable3d = "TRUE" +numvcpus = "2" pciBridge0.present = "TRUE" pciBridge4.functions = "8" pciBridge4.present = "TRUE" @@ -20,13 +26,26 @@ pciBridge6.virtualDev = "pcieRootPort" pciBridge7.functions = "8" pciBridge7.present = "TRUE" pciBridge7.virtualDev = "pcieRootPort" +powerType.powerOff = "soft" +powerType.powerOn = "soft" +powerType.reset = "soft" +powerType.suspend = "soft" +sata0.present = "TRUE" sata0:1.deviceType = "cdrom-raw" +sata0:1.present = "TRUE" +scsi0.present = "TRUE" scsi0.virtualDev = "lsilogic" +serial0.fileName = "thinprint" +serial0.fileType = "thinprint" +serial0.present = "TRUE" sound.autoDetect = "TRUE" sound.fileName = "-1" sound.present = "TRUE" +svga.graphicsMemoryKB = "8388608" tools.syncTime = "FALSE" usb.present = "TRUE" usb.vbluetooth.startConnected = "TRUE" +vcpu.hotadd = "TRUE" virtualHW.productCompatibility = "hosted" virtualHW.version = "18" +vmci0.present = "TRUE" -- cgit v1.2.3-55-g7522