From b56223445acfefd3a954eb3878afe0b92bd316b4 Mon Sep 17 00:00:00 2001 From: Simon Rettberg Date: Wed, 2 Sep 2015 17:54:13 +0200 Subject: Restructure vm meta data parser --- src/main/java/org/openslx/util/vm/DiskImage.java | 85 +++++++ .../java/org/openslx/util/vm/KeyValuePair.java | 13 ++ .../java/org/openslx/util/vm/VmwareConfig.java | 254 +++++++++++++++++++++ .../java/org/openslx/util/vm/VmwareMetaData.java | 212 ++--------------- 4 files changed, 376 insertions(+), 188 deletions(-) create mode 100644 src/main/java/org/openslx/util/vm/DiskImage.java create mode 100644 src/main/java/org/openslx/util/vm/KeyValuePair.java create mode 100644 src/main/java/org/openslx/util/vm/VmwareConfig.java diff --git a/src/main/java/org/openslx/util/vm/DiskImage.java b/src/main/java/org/openslx/util/vm/DiskImage.java new file mode 100644 index 0000000..92138a1 --- /dev/null +++ b/src/main/java/org/openslx/util/vm/DiskImage.java @@ -0,0 +1,85 @@ +package org.openslx.util.vm; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.RandomAccessFile; + +import org.openslx.bwlp.thrift.iface.Virtualizer; + +public class DiskImage +{ + + /** + * Big endian representation of the 4 bytes 'VMDK' + */ + private static final int VMDK_MAGIC = 0x4b444d56; + + public enum ImageFormat + { + VMDK( "vmdk" ), + QCOW2( "qcow2" ), + VDI( "vdi" ); + + public final String extension; + + private ImageFormat( String extension ) + { + this.extension = extension; + } + + public ImageFormat defaultForVirtualizer( Virtualizer virt ) + { + if ( virt == null ) + return null; + if ( virt.virtId.equals( "vmware" ) ) + return VMDK; + if ( virt.virtId.equals( "virtualbox" ) ) + return VDI; + return null; + } + } + + public final boolean isStandalone; + public final boolean isCompressed; + public final ImageFormat format; + + public DiskImage( File disk ) throws FileNotFoundException, IOException, UnknownImageFormatException + { + // For now we only support VMDK... + try ( RandomAccessFile file = new RandomAccessFile( disk, "r" ) ) { + if ( file.read() != VMDK_MAGIC ) + throw new UnknownImageFormatException(); + file.seek( 512 ); + byte[] buffer = new byte[ 2048 ]; + file.readFully( buffer ); + VmwareConfig config = new VmwareConfig( buffer, findNull( buffer ) ); + String ct = config.get( "createType" ); + this.isStandalone = isStandaloneCreateType( ct ); + this.isCompressed = ct != null && ct.equalsIgnoreCase( "streamOptimized" ); + this.format = ImageFormat.VMDK; + } + } + + private int findNull( byte[] buffer ) + { + for ( int i = 0; i < buffer.length; ++i ) { + if ( buffer[i] == 0 ) + return i; + } + return buffer.length; + } + + private boolean isStandaloneCreateType( String type ) + { + if ( type == null ) + return false; + return type.equalsIgnoreCase( "streamOptimized" ) || type.equalsIgnoreCase( "monolithicSparse" ); + } + + public static class UnknownImageFormatException extends Exception + { + private static final long serialVersionUID = -6647935235475007171L; + } + +} diff --git a/src/main/java/org/openslx/util/vm/KeyValuePair.java b/src/main/java/org/openslx/util/vm/KeyValuePair.java new file mode 100644 index 0000000..d89d51b --- /dev/null +++ b/src/main/java/org/openslx/util/vm/KeyValuePair.java @@ -0,0 +1,13 @@ +package org.openslx.util.vm; + +class KeyValuePair +{ + public final String key; + public final String value; + + public KeyValuePair( String key, String value ) + { + this.key = key; + this.value = value; + } +} \ No newline at end of file diff --git a/src/main/java/org/openslx/util/vm/VmwareConfig.java b/src/main/java/org/openslx/util/vm/VmwareConfig.java new file mode 100644 index 0000000..a0fb5bc --- /dev/null +++ b/src/main/java/org/openslx/util/vm/VmwareConfig.java @@ -0,0 +1,254 @@ +package org.openslx.util.vm; + +import java.io.BufferedReader; +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; +import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.TreeMap; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.log4j.Logger; +import org.openslx.util.Util; + +public class VmwareConfig +{ + + private static final Logger LOGGER = Logger.getLogger( VmwareConfig.class ); + + private Map entries = new TreeMap<>( String.CASE_INSENSITIVE_ORDER ); + + public VmwareConfig() + { + // (void) + } + + public VmwareConfig( File file ) throws IOException + { + int todo = (int)Math.min( 100000, file.length() ); + int offset = 0; + byte[] data = new byte[ todo ]; + FileInputStream fr = null; + try { + fr = new FileInputStream( file ); + while ( todo > 0 ) { + int ret = fr.read( data, offset, todo ); + if ( ret <= 0 ) + break; + todo -= ret; + offset += ret; + } + } finally { + Util.safeClose( fr ); + } + init( data, offset ); + + } + + public VmwareConfig( InputStream is ) throws IOException + { + int todo = Math.max( 4000, Math.min( 100000, is.available() ) ); + int offset = 0; + byte[] data = new byte[ todo ]; + while ( todo > 0 ) { + int ret = is.read( data, offset, todo ); + if ( ret <= 0 ) + break; + todo -= ret; + offset += ret; + } + init( data, offset ); + } + + public VmwareConfig( byte[] vmxContent, int length ) + { + init( vmxContent, length ); + } + + private void init( byte[] vmxContent, int length ) + { + String csName = detectCharset( new ByteArrayInputStream( vmxContent, 0, length ) ); + Charset cs = null; + try { + cs = Charset.forName( csName ); + } catch ( Exception e ) { + LOGGER.warn( "Could not instantiate charset " + csName, e ); + } + if ( cs == null ) + cs = StandardCharsets.ISO_8859_1; + try { + BufferedReader reader = new BufferedReader( new InputStreamReader( new ByteArrayInputStream( vmxContent, 0, length ), cs ) ); + String line; + while ( ( line = reader.readLine() ) != null ) { + KeyValuePair entry = parse( line ); + if ( entry != null ) { + set( entry.key, unescape( entry.value ) ); + } + } + } catch ( IOException e ) { + LOGGER.warn( "Exception when loading vmx from byte array (how!?)", e ); + } + } + + private String unescape( String value ) + { + String ret = value; + if ( ret.contains( "|22" ) ) { + ret = ret.replace( "|22", "\"" ); + } + if ( ret.contains( "|7C" ) ) { + ret.replace( "|7C", "|" ); + } + return ret; + } + + private String detectCharset( InputStream is ) + { + try { + BufferedReader csDetectReader = new BufferedReader( new InputStreamReader( is, StandardCharsets.ISO_8859_1 ) ); + String line; + while ( ( line = csDetectReader.readLine() ) != null ) { + KeyValuePair entry = parse( line ); + if ( entry == null ) + continue; + if ( entry.key.equals( ".encoding" ) || entry.key.equals( "encoding" ) ) { + return entry.value; + } + } + } catch ( Exception e ) { + LOGGER.warn( "Could not detect charset, fallback to latin1", e ); + } + // Dumb fallback + return "ISO-8859-1"; + } + + public Set> entrySet() + { + return entries.entrySet(); + } + + private static final Pattern settingMatcher1 = Pattern.compile( "^\\s*(#?[a-z0-9\\.\\:_]+)\\s*=\\s*\"(.*)\"\\s*$", + Pattern.CASE_INSENSITIVE ); + private static final Pattern settingMatcher2 = Pattern.compile( "^\\s*(#?[a-z0-9\\.\\:_]+)\\s*=\\s*([^\"]*)\\s*$", + Pattern.CASE_INSENSITIVE ); + + private KeyValuePair parse( String line ) + { + Matcher matcher = settingMatcher1.matcher( line ); + if ( !matcher.matches() ) { + matcher = settingMatcher2.matcher( line ); + } + if ( !matcher.matches() ) { + return null; + } + return new KeyValuePair( + matcher.group( 1 ), matcher.group( 2 ) ); + + } + + public ConfigEntry set( String key, String value, boolean replace ) + { + if ( !replace && entries.containsKey( key ) ) + return null; + ConfigEntry ce = new ConfigEntry( value ); + entries.put( key, ce ); + return ce; + } + + public ConfigEntry set( String key, String value ) + { + return set( key, value, true ); + } + + public ConfigEntry set( KeyValuePair entry ) + { + return set( entry.key, entry.value ); + } + + public String get( String key ) + { + ConfigEntry ce = entries.get( key ); + if ( ce == null ) + return null; + return ce.value; + } + + public String toString( boolean filteredRequired, boolean generatedRequired ) + { + set( ".encoding", "UTF-8" ); + StringBuilder sb = new StringBuilder( 300 ); + for ( Entry entry : entries.entrySet() ) { + ConfigEntry value = entry.getValue(); + if ( ( !filteredRequired || value.forFiltered ) && + ( !generatedRequired || value.forGenerated ) ) { + sb.append( entry.getKey() ); + sb.append( " = \"" ); + sb.append( value.getEscaped() ); + sb.append( "\"\n" ); + } + } + return sb.toString(); + } + + @Override + public String toString() + { + return toString( false, false ); + } + + public static class ConfigEntry + { + private String value; + private boolean forFiltered; + private boolean forGenerated; + + public ConfigEntry( String value ) + { + this.value = value; + } + + public ConfigEntry filtered( boolean set ) + { + this.forFiltered = set; + return this; + } + + public ConfigEntry generated( boolean set ) + { + this.forGenerated = set; + return this; + } + + public String getEscaped() + { + String ret = value; + if ( ret.contains( "|" ) ) { + ret = ret.replace( "|", "|7C" ); + } + if ( ret.contains( "\"" ) ) { + ret = ret.replace( "\"", "|22" ); + } + return ret; + } + + public String getValue() + { + return value; + } + + public void setValue( String value ) + { + this.value = value; + } + + } + +} diff --git a/src/main/java/org/openslx/util/vm/VmwareMetaData.java b/src/main/java/org/openslx/util/vm/VmwareMetaData.java index 233765b..151f979 100644 --- a/src/main/java/org/openslx/util/vm/VmwareMetaData.java +++ b/src/main/java/org/openslx/util/vm/VmwareMetaData.java @@ -1,13 +1,7 @@ package org.openslx.util.vm; -import java.io.BufferedReader; -import java.io.ByteArrayInputStream; import java.io.File; -import java.io.FileInputStream; import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.List; @@ -19,7 +13,7 @@ import java.util.regex.Pattern; import org.apache.log4j.Logger; import org.openslx.bwlp.thrift.iface.OperatingSystem; import org.openslx.bwlp.thrift.iface.Virtualizer; -import org.openslx.util.Util; +import org.openslx.util.vm.VmwareConfig.ConfigEntry; public class VmwareMetaData extends VmMetaData { @@ -33,6 +27,8 @@ public class VmwareMetaData extends VmMetaData // Lowercase list of allowed settings for upload (as regex) private static final Pattern[] whitelist; + private final VmwareConfig config; + // Init static members static { String[] list = { "^guestos", "^uuid\\.bios", "^config\\.version", "^ehci\\.", @@ -45,73 +41,24 @@ public class VmwareMetaData extends VmMetaData private final Map disks = new HashMap<>(); - private final Map slxLines = new HashMap<>(); - - private final StringBuilder cleanedConfig = new StringBuilder( 200 ); - public VmwareMetaData( List osList, File file ) throws IOException { super( osList ); - int todo = (int)Math.min( 100000, file.length() ); - int offset = 0; - byte[] data = new byte[ todo ]; - FileInputStream fr = null; - try { - fr = new FileInputStream( file ); - while ( todo > 0 ) { - int ret = fr.read( data, offset, todo ); - if ( ret <= 0 ) - break; - todo -= ret; - offset += ret; - } - } finally { - Util.safeClose( fr ); - } - init( data, offset ); + this.config = new VmwareConfig( file ); + init(); } public VmwareMetaData( List osList, byte[] vmxContent, int length ) { super( osList ); - init( vmxContent, length ); + this.config = new VmwareConfig( vmxContent, length ); + init(); } - private void init( byte[] vmxContent, int length ) + private void init() { - String csName = detectCharset( new ByteArrayInputStream( vmxContent, 0, length ) ); - Charset cs = null; - try { - cs = Charset.forName( csName ); - } catch ( Exception e ) { - LOGGER.warn( "Could not instantiate charset " + csName, e ); - } - if ( cs == null ) - cs = StandardCharsets.ISO_8859_1; - try { - loadVmx( new ByteArrayInputStream( vmxContent, 0, length ), cs ); - } catch ( IOException e ) { - LOGGER.warn( "Exception when loading vmx from byte array (how!?)", e ); - } - } - - private void loadVmx( InputStream is, Charset cs ) throws IOException - { - LOGGER.info( "Loading VMX with charset " + cs.name() ); - // Prepare filtered vmx - addFiltered( ".encoding", "UTF-8" ); - // Read line-by-line - BufferedReader reader = null; - try { - reader = new BufferedReader( new InputStreamReader( is, cs ) ); - String line; - while ( ( line = reader.readLine() ) != null ) { - KeyValuePair entry = parse( line ); - if ( entry != null ) - handleLoadEntry( entry ); - } - } finally { - Util.safeClose( reader ); + for ( Entry entry : config.entrySet() ) { + handleLoadEntry( entry ); } // Now find the HDDs and add to list for ( Entry cEntry : disks.entrySet() ) { @@ -148,37 +95,33 @@ public class VmwareMetaData extends VmMetaData private void addFiltered( String key, String value ) { - cleanedConfig.append( key ).append( " = \"" ).append( value ).append( "\"\n" ); + config.set( key, value ).filtered( true ); } - private void handleLoadEntry( KeyValuePair entry ) + private void handleLoadEntry( Entry entry ) { - // Check if it's one of our special keys - if ( entry.key.startsWith( "#SLX_" ) ) { - slxLines.put( entry.key, entry.value ); - return; - } - String lowerKey = entry.key.toLowerCase(); + String lowerKey = entry.getKey().toLowerCase(); // Cleaned vmx construction for ( Pattern exp : whitelist ) { if ( exp.matcher( lowerKey ).find() ) { - addFiltered( entry.key, entry.value ); + entry.getValue().filtered( true ); break; } } // // Dig Usable meta data + String value = entry.getValue().getValue(); if ( lowerKey.equals( "guestos" ) ) { - setOs( "vmware", entry.value ); + setOs( "vmware", value ); return; } if ( lowerKey.equals( "displayname" ) ) { - displayName = entry.value; + displayName = value; return; } - Matcher hdd = hddKey.matcher( entry.key ); + Matcher hdd = hddKey.matcher( entry.getKey() ); if ( hdd.find() ) { - handleHddEntry( hdd.group( 1 ).toLowerCase(), hdd.group( 2 ), hdd.group( 3 ), entry.value ); + handleHddEntry( hdd.group( 1 ).toLowerCase(), hdd.group( 2 ), hdd.group( 3 ), value ); } } @@ -213,52 +156,16 @@ public class VmwareMetaData extends VmMetaData } } - private String detectCharset( InputStream is ) - { - BufferedReader csDetectReader = null; - try { - csDetectReader = new BufferedReader( new InputStreamReader( is, StandardCharsets.ISO_8859_1 ) ); - String line; - while ( ( line = csDetectReader.readLine() ) != null ) { - KeyValuePair entry = parse( line ); - if ( entry == null ) - continue; - if ( entry.key.equals( ".encoding" ) ) { - return entry.value; - } - } - } catch ( Exception e ) { - LOGGER.warn( "Could not detect charset, fallback to latin1", e ); - } finally { - Util.safeClose( csDetectReader ); - } - // Dumb fallback - return "ISO-8859-1"; - } - - private static final Pattern settingMatcher = Pattern.compile( "^\\s*(#?[a-z0-9\\.\\:_]+)\\s*=\\s*\"(.*)\"\\s*$", - Pattern.CASE_INSENSITIVE ); - - private KeyValuePair parse( String line ) - { - Matcher matcher = settingMatcher.matcher( line ); - if ( !matcher.matches() ) - return null; - return new KeyValuePair( - matcher.group( 1 ), matcher.group( 2 ) ); - - } - public boolean addHddTemplate( String diskImagePath ) { DriveBusType bus; try { - bus = DriveBusType.valueOf( slxLines.get( "#SLX_HDD_BUS" ) ); + bus = DriveBusType.valueOf( config.get( "#SLX_HDD_BUS" ) ); } catch ( Exception e ) { - LOGGER.warn( "Unknown bus type: " + slxLines.get( "#SLX_HDD_BUS" ) + ". Cannot add hdd config." ); + LOGGER.warn( "Unknown bus type: " + config.get( "#SLX_HDD_BUS" ) + ". Cannot add hdd config." ); return false; } - String chipset = slxLines.get( "#SLX_HDD_CHIP" ); + String chipset = config.get( "#SLX_HDD_CHIP" ); switch ( bus ) { case IDE: addFiltered( "ide0.present", "TRUE" ); @@ -267,7 +174,7 @@ public class VmwareMetaData extends VmMetaData addFiltered( "ide0:0.fileName", diskImagePath ); return true; case SATA: - // Cannot happen, use lsisas1068 + // Cannot happen?... use lsisas1068 case SCSI: addFiltered( "scsi0.present", "TRUE" ); addFiltered( "scsi0:0.present", "TRUE" ); @@ -305,7 +212,7 @@ public class VmwareMetaData extends VmMetaData @Override public byte[] getFilteredDefinitionArray() { - return cleanedConfig.toString().getBytes( StandardCharsets.UTF_8 ); + return config.toString( true, false ).getBytes( StandardCharsets.UTF_8 ); } @Override @@ -314,77 +221,6 @@ public class VmwareMetaData extends VmMetaData return virtualizer; } - // ############################### - // TODO Structured approach to above mess - - private static class KeyValuePair - { - public final String key; - public final String value; - - public KeyValuePair( String key, String value ) - { - this.key = key; - this.value = value; - } - } - - private static class ConfigEntry - { - private String value; - private boolean forFiltered; - private boolean forGenerated; - - public ConfigEntry( String value ) - { - this.value = value; - } - - public ConfigEntry filtered( boolean set ) - { - this.forFiltered = set; - return this; - } - - public ConfigEntry generated( boolean set ) - { - this.forGenerated = set; - return this; - } - - } - - private static class ConfigEntries - { - private Map entries = new HashMap<>(); - - public ConfigEntry set( String key, String value ) - { - return entries.put( key, new ConfigEntry( value ) ); - } - - public ConfigEntry set( KeyValuePair entry ) - { - return set( entry.key, entry.value ); - } - - public String get( boolean filteredRequired, boolean generatedRequired ) - { - StringBuilder sb = new StringBuilder( 300 ); - for ( Entry entry : entries.entrySet() ) { - ConfigEntry value = entry.getValue(); - if ( ( !filteredRequired || value.forFiltered ) && - ( !generatedRequired || value.forGenerated ) ) { - sb.append( entry.getKey() ); - sb.append( " = \"" ); - sb.append( entry.getValue() ); - sb.append( "\"\n" ); - } - } - return sb.toString(); - } - } - private static class Device { public boolean present = false; -- cgit v1.2.3-55-g7522