Merge branch 'master' into vbox

9 files changed, 132 insertions, 312 deletions

diff --git a/src/main/java/org/openslx/filetransfer/util/ChunkList.java b/src/main/java/org/openslx/filetransfer/util/ChunkList.java
index c692499..cd1bc69 100644
--- a/src/main/java/org/openslx/filetransfer/util/ChunkList.java
+++ b/src/main/java/org/openslx/filetransfer/util/ChunkList.java
@@ -1,5 +1,6 @@
 package org.openslx.filetransfer.util;
 
+import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.Arrays;
@@ -7,6 +8,7 @@ import java.util.Collections;
 import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.zip.CRC32;
 
 import org.apache.log4j.Logger;
 import org.openslx.util.ThriftUtil;
@@ -74,6 +76,36 @@ public class ChunkList
 	}
 
 	/**
+	 * Get CRC32 list in DNBD3 format. All checksums are little
+	 * endian and prefixed by the crc32 sum of the list itself.
+	 */
+	public synchronized byte[] getDnbd3Crc32List() throws IOException
+	{
+		byte buffer[] = new byte[ allChunks.size() * 4 + 4 ]; // 4 byte per chunk plus master
+		long nextChunkOffset = 0;
+		int nextCrcArrayPos = 4;
+		for ( FileChunk c : allChunks ) {
+			if ( c.crc32 == null ) {
+				throw new IllegalStateException( "Called on ChunkList that doesn't have crc32 enabled" );
+			}
+			if ( c.range.startOffset != nextChunkOffset ) {
+				throw new IllegalStateException( "Chunk list is not in order or has wrong chunk size" );
+			}
+			nextChunkOffset += FileChunk.CHUNK_SIZE;
+			c.getCrc32Le( buffer, nextCrcArrayPos );
+			nextCrcArrayPos += 4;
+		}
+		CRC32 masterCrc = new CRC32();
+		masterCrc.update( buffer, 4, buffer.length - 4 );
+		int value = (int)masterCrc.getValue();
+		buffer[3] = (byte) ( value >>> 24 );
+		buffer[2] = (byte) ( value >>> 16 );
+		buffer[1] = (byte) ( value >>> 8 );
+		buffer[0] = (byte)value;
+		return buffer;
+	}
+
+	/**
 	 * Get a missing chunk, marking it pending.
 	 * 
 	 * @return chunk marked as missing
diff --git a/src/main/java/org/openslx/filetransfer/util/FileChunk.java b/src/main/java/org/openslx/filetransfer/util/FileChunk.java
index e00b011..6450af2 100644
--- a/src/main/java/org/openslx/filetransfer/util/FileChunk.java
+++ b/src/main/java/org/openslx/filetransfer/util/FileChunk.java
@@ -2,6 +2,7 @@ package org.openslx.filetransfer.util;
 
 import java.util.Iterator;
 import java.util.List;
+import java.util.zip.CRC32;
 
 import org.openslx.filetransfer.FileRange;
 
@@ -18,6 +19,7 @@ public class FileChunk
 	public final FileRange range;
 	private int failCount = 0;
 	protected byte[] sha1sum;
+	protected CRC32 crc32;
 	protected ChunkStatus status = ChunkStatus.MISSING;
 	private boolean writtenToDisk = false;
 
@@ -73,6 +75,36 @@ public class FileChunk
 		return status;
 	}
 
+	public synchronized void calculateDnbd3Crc32( byte[] data )
+	{
+		// As this is usually called before we validated the sha1, handle the case where
+		// this gets called multiple times and only remember the last result
+		if ( crc32 == null ) {
+			crc32 = new CRC32();
+		} else {
+			crc32.reset();
+		}
+		int chunkLength = range.getLength();
+		crc32.update( data, 0, chunkLength );
+		if ( ( chunkLength % 4096 ) != 0 ) {
+			// DNBD3 virtually pads all images to be a multiple of 4KiB in size,
+			// so simulate that here too
+			byte[] padding = new byte[ 4096 - ( chunkLength % 4096 ) ];
+			crc32.update( padding );
+		}
+	}
+
+	public synchronized void getCrc32Le( byte[] buffer, int offset )
+	{
+		if ( crc32 == null )
+			throw new IllegalStateException( "Trying to get CRC32 on Chunk that doesn't have one" );
+		int value = (int)crc32.getValue();
+		buffer[offset + 3] = (byte) ( value >>> 24 );
+		buffer[offset + 2] = (byte) ( value >>> 16 );
+		buffer[offset + 1] = (byte) ( value >>> 8 );
+		buffer[offset + 0] = (byte)value;
+	}
+
 	/**
 	 * Whether the chunk of data this chunk refers to has been written to
 	 * disk and is assumed to be valid/up to date.
diff --git a/src/main/java/org/openslx/filetransfer/util/HashChecker.java b/src/main/java/org/openslx/filetransfer/util/HashChecker.java
index b9b62b1..273bc7e 100644
--- a/src/main/java/org/openslx/filetransfer/util/HashChecker.java
+++ b/src/main/java/org/openslx/filetransfer/util/HashChecker.java
@@ -12,6 +12,10 @@ import org.apache.log4j.Logger;
 
 public class HashChecker
 {
+	public static final int BLOCKING = 1;
+	public static final int CALC_HASH = 2;
+	public static final int CALC_CRC32 = 4;
+	
 	private static final Logger LOGGER = Logger.getLogger( HashChecker.class );
 
 	private final BlockingQueue<HashTask> queue;
@@ -69,6 +73,8 @@ public class HashChecker
 
 	private void execCallback( HashTask task, HashResult result )
 	{
+		if ( task.callback == null )
+			return;
 		try {
 			task.callback.hashCheckDone( result, task.data, task.chunk );
 		} catch ( Throwable t ) {
@@ -85,12 +91,14 @@ public class HashChecker
 	 * @return true if the chunk was handled, false if the queue was full and rejected the chunk.
 	 * @throws InterruptedException
 	 */
-	public boolean queue( FileChunk chunk, byte[] data, HashCheckCallback callback, boolean blocking ) throws InterruptedException
+	public boolean queue( FileChunk chunk, byte[] data, HashCheckCallback callback, int flags ) throws InterruptedException
 	{
-		byte[] sha1Sum = chunk.getSha1Sum();
-		if ( sha1Sum == null )
+		boolean blocking = ( flags & BLOCKING ) != 0;
+		boolean doHash = ( flags & CALC_HASH ) != 0;
+		boolean doCrc32 = ( flags & CALC_CRC32 ) != 0;
+		if ( doHash && chunk.getSha1Sum() == null )
 			throw new NullPointerException( "Chunk has no sha1 hash" );
-		HashTask task = new HashTask( data, chunk, callback );
+		HashTask task = new HashTask( data, chunk, callback, doHash, doCrc32 );
 		synchronized ( threads ) {
 			if ( invalid ) {
 				execCallback( task, HashResult.FAILURE );
@@ -106,7 +114,9 @@ public class HashChecker
 				}
 			}
 		}
-		chunk.setStatus( ChunkStatus.HASHING );
+		if ( doHash ) {
+			chunk.setStatus( ChunkStatus.HASHING );
+		}
 		if ( blocking ) {
 			queue.put( task );
 		} else {
@@ -153,10 +163,17 @@ public class HashChecker
 					LOGGER.info( "Interrupted while waiting for hash task", e );
 					break;
 				}
-				// Calculate digest
-				md.update( task.data, 0, task.chunk.range.getLength() );
-				byte[] digest = md.digest();
-				HashResult result = Arrays.equals( digest, task.chunk.getSha1Sum() ) ? HashResult.VALID : HashResult.INVALID;
+				HashResult result = HashResult.NONE;
+				if ( task.doHash ) {
+   				// Calculate digest
+   				md.update( task.data, 0, task.chunk.range.getLength() );
+   				byte[] digest = md.digest();
+					result = Arrays.equals( digest, task.chunk.getSha1Sum() ) ? HashResult.VALID : HashResult.INVALID;
+				}
+				if ( task.doCrc32 ) {
+   				// Calculate CRC32
+   				task.chunk.calculateDnbd3Crc32( task.data );
+				}
 				execCallback( task, result );
 				if ( extraThread && queue.isEmpty() ) {
 					break;
@@ -173,6 +190,7 @@ public class HashChecker
 
 	public static enum HashResult
 	{
+		NONE, // No hashing tool place
 		VALID, // Hash matches
 		INVALID, // Hash does not match
 		FAILURE // Error calculating hash
@@ -183,12 +201,16 @@ public class HashChecker
 		public final byte[] data;
 		public final FileChunk chunk;
 		public final HashCheckCallback callback;
+		public final boolean doHash;
+		public final boolean doCrc32;
 
-		public HashTask( byte[] data, FileChunk chunk, HashCheckCallback callback )
+		public HashTask( byte[] data, FileChunk chunk, HashCheckCallback callback, boolean doHash, boolean doCrc32 )
 		{
 			this.data = data;
 			this.chunk = chunk;
 			this.callback = callback;
+			this.doHash = doHash;
+			this.doCrc32 = doCrc32;
 		}
 	}
 
diff --git a/src/main/java/org/openslx/filetransfer/util/IncomingTransferBase.java b/src/main/java/org/openslx/filetransfer/util/IncomingTransferBase.java
index 4135ca7..b298c04 100644
--- a/src/main/java/org/openslx/filetransfer/util/IncomingTransferBase.java
+++ b/src/main/java/org/openslx/filetransfer/util/IncomingTransferBase.java
@@ -214,7 +214,7 @@ public abstract class IncomingTransferBase extends AbstractTransfer implements H
 				continue;
 			}
 			try {
-				if ( !hashChecker.queue( chunk, data, this, false ) ) { // false == queue full, stop
+				if ( !hashChecker.queue( chunk, data, this, HashChecker.CALC_HASH ) ) { // false == queue full, stop
 					chunks.markCompleted( chunk, false );
 					break;
 				}
@@ -285,9 +285,10 @@ public abstract class IncomingTransferBase extends AbstractTransfer implements H
 		public FileRange get()
 		{
 			if ( currentChunk != null ) {
+				chunkReceived( currentChunk, buffer );
 				if ( hashChecker != null && currentChunk.getSha1Sum() != null ) {
 					try {
-						hashChecker.queue( currentChunk, buffer, IncomingTransferBase.this, true );
+						hashChecker.queue( currentChunk, buffer, IncomingTransferBase.this, HashChecker.BLOCKING | HashChecker.CALC_HASH );
 					} catch ( InterruptedException e ) {
 						chunks.markCompleted( currentChunk, false );
 						currentChunk = null;
@@ -515,7 +516,11 @@ public abstract class IncomingTransferBase extends AbstractTransfer implements H
 			return;
 		}
 		try {
-			if ( !hashChecker.queue( chunk, data, this, blocking ) ) {
+			int flags = HashChecker.CALC_HASH;
+			if ( blocking ) {
+				flags |= HashChecker.BLOCKING;
+			}
+			if ( !hashChecker.queue( chunk, data, this, flags ) ) {
 				chunks.markCompleted( chunk, false );
 			}
 		} catch ( InterruptedException e ) {
@@ -540,6 +545,11 @@ public abstract class IncomingTransferBase extends AbstractTransfer implements H
 			}
 		}
 	}
+	
+	protected HashChecker getHashChecker()
+	{
+		return hashChecker;
+	}
 
 	/*
 	 * 
@@ -559,4 +569,11 @@ public abstract class IncomingTransferBase extends AbstractTransfer implements H
 
 	protected abstract void chunkStatusChanged( FileChunk chunk );
 
+	/**
+	 * Called when a chunk has been received -- no validation has taken place yet
+	 */
+	protected void chunkReceived( FileChunk chunk, byte[] data )
+	{
+	}
+
 }
diff --git a/src/main/java/org/openslx/filetransfer/util/StandaloneFileChunk.java b/src/main/java/org/openslx/filetransfer/util/StandaloneFileChunk.java
new file mode 100644
index 0000000..cc47a8e
--- /dev/null
+++ b/src/main/java/org/openslx/filetransfer/util/StandaloneFileChunk.java
@@ -0,0 +1,16 @@
+package org.openslx.filetransfer.util;
+
+public class StandaloneFileChunk extends FileChunk
+{
+
+	public StandaloneFileChunk( long startOffset, long endOffset, byte[] sha1sum )
+	{
+		super( startOffset, endOffset, sha1sum );
+	}
+
+	public void overrideStatus(ChunkStatus status)
+	{
+		this.status = status;
+	}
+	
+}
diff --git a/src/main/java/org/openslx/network/ProxyConfiguration.java b/src/main/java/org/openslx/network/ProxyConfiguration.java
deleted file mode 100644
index ecf800f..0000000
--- a/src/main/java/org/openslx/network/ProxyConfiguration.java
+++ /dev/null
@@ -1,72 +0,0 @@
-package org.openslx.network;
-
-import java.net.Authenticator;
-import java.net.InetSocketAddress;
-import java.net.Proxy;
-import java.net.ProxySelector;
-
-import org.apache.log4j.Logger;
-
-import com.btr.proxy.search.wpad.WpadProxySearchStrategy;
-
-/**
- * Class for configuring proxy settings system wide, if necessary.
- * 
- * @author bjoern
- * 
- */
-public class ProxyConfiguration
-{
-	private static final Logger log = Logger.getLogger( ProxyConfiguration.class );
-
-	public static void configProxy()
-	{
-		// Reset proxy settings first
-		ProxySelector.setDefault( null );
-		Authenticator.setDefault( null );
-
-		// Configuring proxy settings. First read options from config file.
-		ProxyProperties.load();
-		String proxyConfiguration = ProxyProperties.getProxyConf();
-
-		if ( proxyConfiguration.equals( "AUTO" ) || proxyConfiguration.isEmpty() ) {
-			log.info( "Configuring proxy settings automatically..." );
-			// Configuring proxy settings automatically.
-			WpadProxySearchStrategy wPSS = new WpadProxySearchStrategy();
-			try {
-				ProxySelector pS = wPSS.getProxySelector();
-				ProxySelector.setDefault( pS );
-			} catch ( Throwable e ) {
-				log.error( "Setting proxy configuration automatically failed.", e );
-			}
-			return;
-		}
-
-		if ( proxyConfiguration.equals( "YES" ) ) {
-			// Take the proxy settings from config file.
-			// First check if one of the following necessary options might not be set.
-			if ( ProxyProperties.hasProxyAddress() ) {
-				String proxyAddress = ProxyProperties.getProxyAddress();
-				int proxyPort = ProxyProperties.getProxyPort();
-
-				// Configure proxy.
-				Proxy proxy = new Proxy( Proxy.Type.SOCKS, new InetSocketAddress( proxyAddress, proxyPort ) );
-				StaticProxySelector sPS = new StaticProxySelector( proxy );
-				ProxySelector.setDefault( sPS );
-
-				if ( !ProxyProperties.hasProxyCredentials() ) {
-					log.info( "Configuring proxy settings manually WITH authentication..." );
-					// use Proxy with authentication.
-					String proxyUname = ProxyProperties.getProxyUsername();
-					String proxyPass = ProxyProperties.getProxyPassword();
-
-					// Set authentication.
-					StaticProxyAuthenticator sPA = new StaticProxyAuthenticator( proxyUname, proxyPass );
-					Authenticator.setDefault( sPA );
-				}
-			}
-		}
-
-	}
-
-}
diff --git a/src/main/java/org/openslx/network/ProxyProperties.java b/src/main/java/org/openslx/network/ProxyProperties.java
deleted file mode 100644
index 5dacc57..0000000
--- a/src/main/java/org/openslx/network/ProxyProperties.java
+++ /dev/null
@@ -1,89 +0,0 @@
-package org.openslx.network;
-
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.nio.charset.StandardCharsets;
-import java.util.Properties;
-
-import org.apache.log4j.Logger;
-import org.openslx.util.Util;
-
-public class ProxyProperties
-{
-	private static Logger log = Logger.getLogger( ProxyProperties.class );
-	private static final Properties properties = new Properties();
-
-	// Getting the proxy settings from config file stored in
-	// "/opt/openslx/proxy/conf".
-	public static String getProxyConf()
-	{
-		return properties.getProperty( "PROXY_CONF", "" );
-	}
-
-	public static String getProxyAddress()
-	{
-		return properties.getProperty( "PROXY_ADDR", "" );
-	}
-
-	public static String getProxyUsername()
-	{
-		return properties.getProperty( "PROXY_USERNAME", "" );
-	}
-
-	public static String getProxyPassword()
-	{
-		return properties.getProperty( "PROXY_PASSWORD", "" );
-	}
-
-	// Integers //
-	public static int getProxyPort()
-	{
-		return Util.parseInt( properties.getProperty( "PROXY_PORT", "0" ), 0 );
-	}
-
-	static
-	{
-		load();
-	}
-
-	/**
-	 * Load properties
-	 */
-	public static void load()
-	{
-		InputStreamReader stream = null;
-		try {
-			properties.clear();
-			// Load all entries of the config file into properties
-			stream = new InputStreamReader(
-					new FileInputStream( "/opt/openslx/proxy/config" ), StandardCharsets.UTF_8 );
-			properties.load( stream );
-			stream.close();
-		} catch ( IOException e ) {
-			log.warn( "Could not load proxy properties from '/opt/openslx/proxy/conf'." );
-		} finally {
-			Util.safeClose( stream );
-		}
-	}
-
-	/**
-	 * Check proxy settings for being not empty.
-	 * 
-	 * @return true if address and port are set
-	 */
-	public static boolean hasProxyAddress()
-	{
-		return !getProxyAddress().isEmpty() && getProxyPort() != 0;
-	}
-
-	/**
-	 * Check if a username or password is configured.
-	 * 
-	 * @return true if either username or password (or both) are set
-	 */
-	public static boolean hasProxyCredentials()
-	{
-		return !getProxyUsername().isEmpty() || !getProxyPassword().isEmpty();
-	}
-}
diff --git a/src/main/java/org/openslx/network/StaticProxyAuthenticator.java b/src/main/java/org/openslx/network/StaticProxyAuthenticator.java
deleted file mode 100644
index c1d8da7..0000000
--- a/src/main/java/org/openslx/network/StaticProxyAuthenticator.java
+++ /dev/null
@@ -1,23 +0,0 @@
-package org.openslx.network;
-
-import java.net.Authenticator;
-import java.net.PasswordAuthentication;
-
-public class StaticProxyAuthenticator extends Authenticator
-{
-	private final String username, password;
-
-	public StaticProxyAuthenticator( String username, String password )
-	{
-		this.username = username;
-		this.password = password;
-	}
-
-	protected PasswordAuthentication getPasswordAuthentication()
-	{
-		if ( getRequestorType() != RequestorType.PROXY )
-			return super.getPasswordAuthentication();
-		return new PasswordAuthentication(
-				this.username, this.password.toCharArray() );
-	}
-}
diff --git a/src/main/java/org/openslx/network/StaticProxySelector.java b/src/main/java/org/openslx/network/StaticProxySelector.java
deleted file mode 100644
index d7d76ac..0000000
--- a/src/main/java/org/openslx/network/StaticProxySelector.java
+++ /dev/null
@@ -1,115 +0,0 @@
-package org.openslx.network;
-
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.NetworkInterface;
-import java.net.Proxy;
-import java.net.ProxySelector;
-import java.net.SocketAddress;
-import java.net.SocketException;
-import java.net.URI;
-import java.util.ArrayList;
-import java.util.Enumeration;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-import org.apache.log4j.Logger;
-
-public class StaticProxySelector extends ProxySelector
-{
-	private static Logger log = Logger.getLogger( StaticProxySelector.class );
-
-	private final Proxy proxy;
-	private Set<String> localAddresses = null;
-	private long nextAddressGet = 0;
-
-	public StaticProxySelector( Proxy proxy )
-	{
-		this.proxy = proxy;
-	}
-
-	@Override
-	public void connectFailed( URI uri, SocketAddress sa, IOException ioe )
-	{
-		// Just one fix proxy. So no code is necessary here for deactivating proxy.
-	}
-
-	@Override
-	public List<Proxy> select( URI uri )
-	{
-		List<Proxy> proxyList = new ArrayList<Proxy>();
-
-		String host = uri.getHost();
-		if ( host == null ) // Host not set? Well, we can only guess then, so try to use the proxy
-			return proxyList;
-
-		host = host.replaceFirst( "%\\d+$", "" );
-		if ( host.equals( "localhost" ) || host.startsWith( "127." )
-				|| host.startsWith( "::1" ) || host.startsWith( "0:0:0:0:0:0:0:1" ) ) // Localhost = no proxy
-			return proxyList;
-
-		final Set<String> addrs;
-		synchronized ( this ) {
-			addrs = getLocalAddresses();
-		}
-		if ( !addrs.contains( host ) ) {
-			proxyList.add( this.proxy );
-		}
-
-		return proxyList;
-	}
-
-	/**
-	 * Get all local (IP) addresses
-	 * 
-	 * @return
-	 */
-	private Set<String> getLocalAddresses()
-	{
-		long now = System.currentTimeMillis();
-		if ( now < nextAddressGet )
-			return localAddresses;
-		nextAddressGet = now + 60000;
-
-		List<NetworkInterface> interfaces = getNetworkInterfaces();
-		if ( interfaces == null )
-			return localAddresses; // Fallback on last known data
-		// iterate over network interfaces and get all addresses
-		Set<String> addrs = new HashSet<>();
-		for ( NetworkInterface iface : interfaces ) {
-			Enumeration<InetAddress> e = iface.getInetAddresses();
-			// iterate over InetAddresses of current interface
-			while ( e.hasMoreElements() ) {
-				addrs.add( e.nextElement().getHostAddress().replaceFirst( "%\\d+$", "" ) );
-			}
-		}
-		synchronized ( this ) {
-			localAddresses = addrs;
-		}
-		return localAddresses;
-	}
-
-	/**
-	 * Get a list of all local network interfaces
-	 * 
-	 * @return
-	 */
-	private List<NetworkInterface> getNetworkInterfaces()
-	{
-		ArrayList<NetworkInterface> retList = new ArrayList<NetworkInterface>();
-		Enumeration<NetworkInterface> e = null;
-		try {
-			e = NetworkInterface.getNetworkInterfaces();
-		} catch ( SocketException e1 ) {
-			// TODO Auto-generated catch block
-			e1.printStackTrace();
-			return null;
-		}
-		while ( e.hasMoreElements() ) {
-			retList.add( e.nextElement() );
-		}
-		return retList;
-	}
-
-}