Fix some bugs while testing the connection

Change log level of thrift filetransfer to error in log4j properties

7 files changed, 126 insertions, 47 deletions

diff --git a/src/main/java/org/openslx/imagemaster/App.java b/src/main/java/org/openslx/imagemaster/App.java
index bf66428..b108576 100644
--- a/src/main/java/org/openslx/imagemaster/App.java
+++ b/src/main/java/org/openslx/imagemaster/App.java
@@ -4,6 +4,7 @@ import java.util.ArrayList;
 import java.util.List;
 
 import org.apache.log4j.Logger;
+import org.openslx.imagemaster.serverconnection.CRCScheduler;
 import org.openslx.imagemaster.serverconnection.ImageProcessor;
 import org.openslx.imagemaster.thrift.server.BinaryListener;
 import org.slf4j.LoggerFactory;
@@ -36,6 +37,9 @@ public class App
 		servers.add( t );
 		t.start();
 		
+		// start the crc checking scheduler
+		CRCScheduler.startScheduling();
+		
 		// Run more servers
 		// ...
 		// Wait for all servers to die
diff --git a/src/main/java/org/openslx/imagemaster/db/DbImage.java b/src/main/java/org/openslx/imagemaster/db/DbImage.java
index 38ca714..386c509 100644
--- a/src/main/java/org/openslx/imagemaster/db/DbImage.java
+++ b/src/main/java/org/openslx/imagemaster/db/DbImage.java
@@ -77,7 +77,7 @@ public class DbImage
 		for ( int i : blockStatus ) {
 			blockStatus[i] = UploadingImage.valid;
 		}
-		for ( int i = 0; i < parts.length - 1; i++ ) {		// do not copy the last empty string	(1;2;3;) -> "1","2","3",""
+		for ( int i = 0; i < parts.length; i++ ) {		// do not copy the last empty string	(1;2;3;) -> "1","2","3",""
 			blockStatus[i] = UploadingImage.missing;
 		}
 
diff --git a/src/main/java/org/openslx/imagemaster/serverconnection/CRCScheduler.java b/src/main/java/org/openslx/imagemaster/serverconnection/CRCScheduler.java
index f088af4..c47eb04 100644
--- a/src/main/java/org/openslx/imagemaster/serverconnection/CRCScheduler.java
+++ b/src/main/java/org/openslx/imagemaster/serverconnection/CRCScheduler.java
@@ -6,6 +6,7 @@ import java.util.List;
 import java.util.Timer;
 import java.util.TimerTask;
 
+import org.apache.log4j.Logger;
 import org.openslx.imagemaster.crcchecker.CRCChecker;
 
 /**
@@ -14,28 +15,39 @@ import org.openslx.imagemaster.crcchecker.CRCChecker;
 public class CRCScheduler extends TimerTask
 {
 
+	private static Logger log = Logger.getLogger( CRCScheduler.class );
+
 	@Override
 	public void run()
 	{
+		log.debug( "Starting checks..." );
 		List<UploadingImage> list = ImageProcessor.getImagesToCheck();
+		log.debug(list);
 		Iterator<UploadingImage> iter = list.iterator();
 		while ( iter.hasNext() ) {
 			UploadingImage image = iter.next();
+			log.debug( "Checking blocks of " + image.getDbImage().imageName );
 			CRCChecker crcChecker = new CRCChecker( image.getImageFile(), image.getCrcFile() );
+			log.debug( "CRCFile is valid: " + crcChecker.hasValidCrcFile() );
 			for ( int block = 0; block < image.getNumberOfBlocks(); block++ ) {
 				if ( image.needsCheck( block ) ) {
 					try {
 						if ( crcChecker.checkBlock( block ) ) {
 							image.setValid( block );
+							log.debug( block + " was valid" );
 						} else {
 							image.setNeedsRequest( block );
+							log.debug( block + " was NOT valid" );
 						}
 					} catch ( IOException e ) {
 						// TODO: Handle that crc file or image file could not be read.
+						log.debug( "error" );
 					}
 				}
 			}
+			image.updateDb();
 		}
+		log.debug( "... done" );
 	}
 
 	public static void startScheduling()
@@ -43,7 +55,9 @@ public class CRCScheduler extends TimerTask
 		Timer timer = new Timer( "CRCScheduler" );
 
 		// start now and fire every 60 s
-		timer.schedule( new CRCScheduler(), 0, 60000 );
+		//timer.schedule( new CRCScheduler(), 0, 60000L );
+
+		timer.schedule( new CRCScheduler(), 0L, 30000L );
 	}
 
 }
diff --git a/src/main/java/org/openslx/imagemaster/serverconnection/ConnectionHandler.java b/src/main/java/org/openslx/imagemaster/serverconnection/ConnectionHandler.java
index 546094a..b35088b 100644
--- a/src/main/java/org/openslx/imagemaster/serverconnection/ConnectionHandler.java
+++ b/src/main/java/org/openslx/imagemaster/serverconnection/ConnectionHandler.java
@@ -1,5 +1,6 @@
 package org.openslx.imagemaster.serverconnection;
 
+import java.io.File;
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;
 import java.io.IOException;
@@ -115,7 +116,8 @@ public class ConnectionHandler implements IncomingEvent
 	public void incomingUploader( Uploader uploader ) throws IOException
 	{
 		// try to read meta data
-		if ( !uploader.readMetaData() ) return;
+		if ( !uploader.readMetaData() )
+			return;
 		String token = uploader.getToken();
 		log.debug( "Got token :'" + token + "'" );
 		// check token to identify the client
@@ -131,14 +133,18 @@ public class ConnectionHandler implements IncomingEvent
 			uploader.close();
 			return;
 		}
-		
+
 		List<Integer> blocks = connections.get( token ).client.getLastRequestedBlocks( token );
+		String fileName = connections.get( token ).filepath;
+		long fileSize = new File( fileName ).length();
+		long actual;
 		for ( Integer block : blocks ) {
-			uploader.sendRange( block*Globals.blockSize, (block+1)*Globals.blockSize - 1 );
-			uploader.sendFile( connections.get( token ).filepath );
-			uploader.close();
+			actual = block * Globals.blockSize;
+			uploader.sendRange( (int)actual, ( ( fileSize - actual ) < Globals.blockSize ) ? (int)fileSize - 1 : ( block + 1 ) * Globals.blockSize - 1 );	// TODO: change this int into long
+			uploader.sendFile( fileName );
 		}
-		
+		uploader.close();
+
 	}
 
 	/**
@@ -147,9 +153,8 @@ public class ConnectionHandler implements IncomingEvent
 	@Override
 	public void incomingDownloader( Downloader downloader ) throws IOException
 	{
+		log.debug( "Client wants to upload" );
 		int startOfRange = 0;
-		int endOfRange = 0;
-		int diffOfRange = 0;
 		String token = "";
 		// try to read meta data
 		while ( downloader.readMetaData() ) {
@@ -161,10 +166,11 @@ public class ConnectionHandler implements IncomingEvent
 				return;
 			}
 
-			// get range
 			startOfRange = downloader.getStartOfRange();
-			endOfRange = downloader.getEndOfRange();
-			diffOfRange = downloader.getDiffOfRange();
+			
+			if ( downloader.getDiffOfRange() <= 0 ) {
+				return;
+			}
 
 			// check if he was a uploading client
 			if ( connections.get( token ).type == Connection.DOWNLOADING ) {
@@ -172,19 +178,15 @@ public class ConnectionHandler implements IncomingEvent
 				downloader.close();
 				return;
 			}
+			
+			int blockNumber = startOfRange / Globals.blockSize;
+			UploadingImage image = connections.get( token ).image;
+			image.setNeedsCheck( blockNumber );
+			image.increaseTransmittedTimes( blockNumber );
+			log.debug( "Block " + blockNumber + " was transmitted " + image.getTimesTransmitted( blockNumber ) + " time(s)." );
 
 			downloader.setOutputFilename( connections.get( token ).filepath );
 			downloader.readBinary();
-			
-			// calculate and register the incoming blocks
-			if ( diffOfRange == 0 )
-				return;
-			
-			for ( int i = startOfRange / Globals.blockSize; i < endOfRange / Globals.blockSize; i += Globals.blockSize ) {
-				UploadingImage image = connections.get( token ).image;
-				image.setNeedsCheck( i );
-				image.increaseTransmittedTimes( i );
-			}
 		}
 		downloader.close();
 	}
diff --git a/src/main/java/org/openslx/imagemaster/serverconnection/ImageProcessor.java b/src/main/java/org/openslx/imagemaster/serverconnection/ImageProcessor.java
index f7f9faa..a27d543 100644
--- a/src/main/java/org/openslx/imagemaster/serverconnection/ImageProcessor.java
+++ b/src/main/java/org/openslx/imagemaster/serverconnection/ImageProcessor.java
@@ -1,7 +1,6 @@
 package org.openslx.imagemaster.serverconnection;
 
 import java.io.IOException;
-import java.sql.Timestamp;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.Iterator;
@@ -69,9 +68,7 @@ public class ImageProcessor
 			throws UploadException, ImageDataException
 	{
 		// check image data
-		if ( DbImage.exists( imageData.uuid ) ) {
-			throw new ImageDataException( ImageDataError.INVALID_DATA, "UUID already existing." );
-		} else if ( imageData.imageName == null || imageData.imageName.isEmpty() ) {
+		if ( imageData.imageName == null || imageData.imageName.isEmpty() ) {
 			throw new ImageDataException( ImageDataError.INVALID_DATA, "Image name not set." );
 		} else if ( imageData.imageName == null || imageData.imageOwner.isEmpty() ) {
 			throw new ImageDataException( ImageDataError.INVALID_DATA, "Image owner not set." );
@@ -83,6 +80,8 @@ public class ImageProcessor
 			throw new ImageDataException( ImageDataError.INVALID_DATA, "User is not known." );
 		}
 
+		log.debug( serverSessionId + " is submitting " + imageData.uuid );
+
 		String uuid = imageData.uuid;
 		String token;
 		String filepath;
@@ -95,19 +94,22 @@ public class ImageProcessor
 			if ( ( image = uploadingImages.get( uuid ) ) != null ) {
 				log.debug( "Image is already uploading.. returning some missing blocks" );
 				List<Integer> missing = getNMissingBlocks( image, AMOUNT );
-				if ( missing.isEmpty() ) {
+				if ( missing.isEmpty() && image.allBlocksValid() ) {
 					uploadDone( uuid );
-					return new UploadInfos( null, 0, missing );
+					return new UploadInfos( image.getToken(), Globals.getSslSocketPort(), missing, image.allBlocksValid() );
 				}
-				return new UploadInfos( image.getToken(), Globals.getSslSocketPort(), missing );
+				return new UploadInfos( image.getToken(), Globals.getSslSocketPort(), missing, false );
 			}
 
 			// insert new image
+			if ( DbImage.exists( imageData.uuid ) ) {
+				throw new ImageDataException( ImageDataError.INVALID_DATA, "UUID already existing." );
+			}
 			if ( !CRCFile.sumsAreValid( crcSums ) )
 				throw new UploadException( UploadError.INVALID_CRC, "CRC sums were invalid." );
 			filepath = Globals.getImageDir() + "/" + uuid + ".vmdk";
 			token = RandomString.generate( 100, false );
-			nBlocks = (int)Math.ceil( imageData.fileSize / Globals.blockSize );
+			nBlocks = crcSums.size() - 1;
 			allBlocks = new int[ nBlocks ];	// initalize array with all zeros which mean that this block is missing
 			image = new UploadingImage( token, allBlocks, System.currentTimeMillis(), uuid, filepath );
 			uploadingImages.put( uuid, image );
@@ -126,9 +128,11 @@ public class ImageProcessor
 		ConnectionHandler.addConnection( token, filepath, Connection.UPLOADING ).image = image;
 		DbImage.insert( imageData, System.currentTimeMillis(), token, nBlocks, serverSessionId, filepath );
 		imagesToCheck.add( uuid );
+		log.debug( imagesToCheck );
 
 		log.debug( "Returning UploadInfos. Client should goint to start the upload. " );
-		return new UploadInfos( token, Globals.getSslSocketPort(), getNMissingBlocks( image, AMOUNT ) );
+		log.debug( image.toString() );
+		return new UploadInfos( token, Globals.getSslSocketPort(), getNMissingBlocks( image, AMOUNT ), false );
 	}
 
 	public static DownloadInfos getDownloadInfos( String serverSessionId, String uuid, List<Integer> requestedBlocks )
@@ -198,19 +202,22 @@ public class ImageProcessor
 	 */
 	private static List<Integer> getNMissingBlocks( UploadingImage image, int amount ) throws UploadException
 	{
-		int size = image.getAmountOfMissingBlocks();
-		if ( amount > size )
-			amount = size;
+		int missing = image.getAmountOfBlocksNeedingRequest();
+		log.debug( "The number of missing blocks: " + missing );
+		if ( amount > missing )
+			amount = missing;
 
 		List<Integer> result = new ArrayList<>( amount );
 
 		int got = 0;
 		for ( int i = 0; i < image.getNumberOfBlocks(); i++ ) {
+			log.debug( i );
 			if ( image.needsRequest( i ) ) {
 				int times = image.getTimesTransmitted( i );
-				if (times > 20) {			// TODO: make configurable
-					throw new UploadException(UploadError.BROKEN_BLOCK, "Block " + i + " was transmitted "
-							+ times + " and is still not valid.");
+				if ( times > 20 ) {			// TODO: make configurable
+					log.debug( "Block " + i + " is probably broken." );
+					throw new UploadException( UploadError.BROKEN_BLOCK, "Block " + i + " was transmitted "
+							+ times + " and is still not valid." );
 				}
 				result.add( i );
 				got++;
@@ -219,6 +226,8 @@ public class ImageProcessor
 				break;
 		}
 
+		log.debug( "Returned " + got + " missing blocks." );
+
 		return result;
 	}
 
@@ -232,6 +241,7 @@ public class ImageProcessor
 	{
 		synchronized ( imagesToCheck ) {
 			imagesToCheck.remove( uuid );
+			log.debug( "Removing " + uuid );
 		}
 
 		UploadingImage image;
@@ -248,8 +258,10 @@ public class ImageProcessor
 	{
 		List<UploadingImage> result = new LinkedList<>();
 		Iterator<String> iter = imagesToCheck.iterator();
+		log.debug( imagesToCheck );
 		while ( iter.hasNext() ) {
 			result.add( uploadingImages.get( iter.next() ) );
+			log.debug( "Added an image ... " );
 		}
 		return result;
 	}
@@ -268,9 +280,9 @@ public class ImageProcessor
 		List<DbImage> list = DbImage.getUploadingImages();
 		for ( DbImage image : list ) {
 			String token = image.token;
-			ConnectionHandler.addConnection( token, image.imagePath, Connection.UPLOADING );
 			UploadingImage infos = new UploadingImage( token, image.blockStatus, image.timestamp.getTime(),
 					image.uuid, image.imagePath );
+			ConnectionHandler.addConnection( token, image.imagePath, Connection.UPLOADING ).image = infos;
 			CRCFile crcFile = new CRCFile( Globals.getImageDir() + "/" + image.uuid + ".crc" );
 			try {
 				if ( !crcFile.isValid() )
@@ -280,6 +292,7 @@ public class ImageProcessor
 			}
 			infos.setCrcFile( crcFile );
 			uploadingImages.put( image.uuid, infos );
+			imagesToCheck.add( image.uuid );
 		}
 		log.info( "Added " + list.size() + " pending upload(s) to process list again." );
 	}
diff --git a/src/main/java/org/openslx/imagemaster/serverconnection/UploadingImage.java b/src/main/java/org/openslx/imagemaster/serverconnection/UploadingImage.java
index dc15dae..8cbaecf 100644
--- a/src/main/java/org/openslx/imagemaster/serverconnection/UploadingImage.java
+++ b/src/main/java/org/openslx/imagemaster/serverconnection/UploadingImage.java
@@ -1,5 +1,8 @@
 package org.openslx.imagemaster.serverconnection;
 
+import java.util.ArrayList;
+import java.util.List;
+
 import org.apache.log4j.Logger;
 import org.openslx.imagemaster.Globals;
 import org.openslx.imagemaster.crcchecker.CRCFile;
@@ -40,6 +43,7 @@ public class UploadingImage
 		this.timestamp = timestamp;
 		this.uuid = uuid;
 		this.blockStatus = initialBlockStatus;
+		this.filename = filename;
 	}
 
 	protected void setValid( int index )
@@ -48,6 +52,21 @@ public class UploadingImage
 			return;
 		blockStatus[index] = valid;
 	}
+	
+	protected void updateDb()
+	{
+		if ( blockStatus == null )
+			return;
+		
+		List<Integer> missingBlocks =  new ArrayList<>();
+		
+		for (int block = 0; block < blockStatus.length; block++) {
+			if (blockStatus[block] != valid) {
+				missingBlocks.add( block );
+			}
+		}
+		dbImage.updateMissingBlocks( missingBlocks );
+	}
 
 	protected void setMissing( int index )
 	{
@@ -72,9 +91,9 @@ public class UploadingImage
 
 	protected void increaseTransmittedTimes( int index )
 	{
-		if ( blockStatus == null || blockStatus[index] == 0 || blockStatus[index] == 200 )
+		if ( blockStatus == null || blockStatus[index] == 200 )
 			return;
-		blockStatus[index] += ( blockStatus[index] < missing ) ? -1 : 1;	// increase in both directions
+		blockStatus[index] += ( blockStatus[index] <= missing ) ? -1 : 1;	// increase in both directions
 	}
 
 	protected int getTimesTransmitted( int index )
@@ -91,7 +110,7 @@ public class UploadingImage
 	{
 		if ( blockStatus == null )
 			return false;
-		return ( blockStatus[index] > missing && blockStatus[index] != valid );
+		return ( ( blockStatus[index] >= missing ) && ( blockStatus[index] != valid ) );
 	}
 
 	protected boolean needsCheck( int index )
@@ -103,6 +122,13 @@ public class UploadingImage
 
 	protected int getNumberOfBlocks()
 	{
+		///////////////////////////////////////////////////////////////////
+		ArrayList<Integer> l = new ArrayList<Integer>( blockStatus.length );
+		for ( int i : blockStatus ) {
+			l.add( i );
+		}
+		log.debug( l );
+		///////////////////////////////////////////////////////////////////
 		return blockStatus.length;
 	}
 
@@ -137,15 +163,34 @@ public class UploadingImage
 		this.crcFile = crcFile;
 	}
 
-	public int getAmountOfMissingBlocks()
+	public int getAmountOfBlocksNeedingRequest()
 	{
 		if ( blockStatus == null )
 			return 0;
 		int result = 0;
-		for ( int i : blockStatus ) {
-			if ( blockStatus[i] == missing )
+		for ( int i = 0; i < blockStatus.length; i++ ) {
+			if ( needsRequest( i ) )
 				result++;
 		}
 		return result;
 	}
+	
+	public boolean allBlocksValid()
+	{
+		if ( blockStatus == null )
+			return false;
+		for ( int i : blockStatus ) {
+			if ( i != 200 )
+				return false;
+		}
+		return true;
+	}
+
+	@Override
+	public String toString()
+	{
+		return "UUID: " + uuid + ", filename " + filename + "\nmissing blocks " + getAmountOfBlocksNeedingRequest() +
+				", number of blocks " + getNumberOfBlocks() + ", token " + getToken();
+
+	}
 }
diff --git a/src/main/properties/log4j.properties b/src/main/properties/log4j.properties
index ced1f8d..1a15589 100644
--- a/src/main/properties/log4j.properties
+++ b/src/main/properties/log4j.properties
@@ -4,8 +4,9 @@ log4j.appender.A1=org.apache.log4j.ConsoleAppender
 
 log4j.appender.A1.layout=org.apache.log4j.PatternLayout
 
-log4j.appender.A1.layout.ConversionPattern=%d [%t] %-5p %c - %m%n
+log4j.appender.A1.layout.ConversionPattern=%d [%t] %-5p %F - %m%n
 
 log4j.logger.org.apache.directory=ERROR
 log4j.logger.org.apache.mina=ERROR
-log4j.logger.org.apache.ftpserver=ERROR
\ No newline at end of file
+log4j.logger.org.apache.ftpserver=ERROR
+log4j.logger.org.openslx.filetransfer=ERROR
\ No newline at end of file