From 56512f44aa6430aeeabe5b81057d30c169fce104 Mon Sep 17 00:00:00 2001 From: Nils Schwabe Date: Wed, 23 Jul 2014 14:19:46 +0200 Subject: Fix some bugs while testing the connection Change log level of thrift filetransfer to error in log4j properties --- src/main/java/org/openslx/imagemaster/App.java | 4 ++ .../java/org/openslx/imagemaster/db/DbImage.java | 2 +- .../imagemaster/serverconnection/CRCScheduler.java | 16 +++++- .../serverconnection/ConnectionHandler.java | 44 +++++++++-------- .../serverconnection/ImageProcessor.java | 45 +++++++++++------ .../serverconnection/UploadingImage.java | 57 +++++++++++++++++++--- src/main/properties/log4j.properties | 5 +- 7 files changed, 126 insertions(+), 47 deletions(-) diff --git a/src/main/java/org/openslx/imagemaster/App.java b/src/main/java/org/openslx/imagemaster/App.java index bf66428..b108576 100644 --- a/src/main/java/org/openslx/imagemaster/App.java +++ b/src/main/java/org/openslx/imagemaster/App.java @@ -4,6 +4,7 @@ import java.util.ArrayList; import java.util.List; import org.apache.log4j.Logger; +import org.openslx.imagemaster.serverconnection.CRCScheduler; import org.openslx.imagemaster.serverconnection.ImageProcessor; import org.openslx.imagemaster.thrift.server.BinaryListener; import org.slf4j.LoggerFactory; @@ -36,6 +37,9 @@ public class App servers.add( t ); t.start(); + // start the crc checking scheduler + CRCScheduler.startScheduling(); + // Run more servers // ... // Wait for all servers to die diff --git a/src/main/java/org/openslx/imagemaster/db/DbImage.java b/src/main/java/org/openslx/imagemaster/db/DbImage.java index 38ca714..386c509 100644 --- a/src/main/java/org/openslx/imagemaster/db/DbImage.java +++ b/src/main/java/org/openslx/imagemaster/db/DbImage.java @@ -77,7 +77,7 @@ public class DbImage for ( int i : blockStatus ) { blockStatus[i] = UploadingImage.valid; } - for ( int i = 0; i < parts.length - 1; i++ ) { // do not copy the last empty string (1;2;3;) -> "1","2","3","" + for ( int i = 0; i < parts.length; i++ ) { // do not copy the last empty string (1;2;3;) -> "1","2","3","" blockStatus[i] = UploadingImage.missing; } diff --git a/src/main/java/org/openslx/imagemaster/serverconnection/CRCScheduler.java b/src/main/java/org/openslx/imagemaster/serverconnection/CRCScheduler.java index f088af4..c47eb04 100644 --- a/src/main/java/org/openslx/imagemaster/serverconnection/CRCScheduler.java +++ b/src/main/java/org/openslx/imagemaster/serverconnection/CRCScheduler.java @@ -6,6 +6,7 @@ import java.util.List; import java.util.Timer; import java.util.TimerTask; +import org.apache.log4j.Logger; import org.openslx.imagemaster.crcchecker.CRCChecker; /** @@ -14,28 +15,39 @@ import org.openslx.imagemaster.crcchecker.CRCChecker; public class CRCScheduler extends TimerTask { + private static Logger log = Logger.getLogger( CRCScheduler.class ); + @Override public void run() { + log.debug( "Starting checks..." ); List list = ImageProcessor.getImagesToCheck(); + log.debug(list); Iterator iter = list.iterator(); while ( iter.hasNext() ) { UploadingImage image = iter.next(); + log.debug( "Checking blocks of " + image.getDbImage().imageName ); CRCChecker crcChecker = new CRCChecker( image.getImageFile(), image.getCrcFile() ); + log.debug( "CRCFile is valid: " + crcChecker.hasValidCrcFile() ); for ( int block = 0; block < image.getNumberOfBlocks(); block++ ) { if ( image.needsCheck( block ) ) { try { if ( crcChecker.checkBlock( block ) ) { image.setValid( block ); + log.debug( block + " was valid" ); } else { image.setNeedsRequest( block ); + log.debug( block + " was NOT valid" ); } } catch ( IOException e ) { // TODO: Handle that crc file or image file could not be read. + log.debug( "error" ); } } } + image.updateDb(); } + log.debug( "... done" ); } public static void startScheduling() @@ -43,7 +55,9 @@ public class CRCScheduler extends TimerTask Timer timer = new Timer( "CRCScheduler" ); // start now and fire every 60 s - timer.schedule( new CRCScheduler(), 0, 60000 ); + //timer.schedule( new CRCScheduler(), 0, 60000L ); + + timer.schedule( new CRCScheduler(), 0L, 30000L ); } } diff --git a/src/main/java/org/openslx/imagemaster/serverconnection/ConnectionHandler.java b/src/main/java/org/openslx/imagemaster/serverconnection/ConnectionHandler.java index 546094a..b35088b 100644 --- a/src/main/java/org/openslx/imagemaster/serverconnection/ConnectionHandler.java +++ b/src/main/java/org/openslx/imagemaster/serverconnection/ConnectionHandler.java @@ -1,5 +1,6 @@ package org.openslx.imagemaster.serverconnection; +import java.io.File; import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; @@ -115,7 +116,8 @@ public class ConnectionHandler implements IncomingEvent public void incomingUploader( Uploader uploader ) throws IOException { // try to read meta data - if ( !uploader.readMetaData() ) return; + if ( !uploader.readMetaData() ) + return; String token = uploader.getToken(); log.debug( "Got token :'" + token + "'" ); // check token to identify the client @@ -131,14 +133,18 @@ public class ConnectionHandler implements IncomingEvent uploader.close(); return; } - + List blocks = connections.get( token ).client.getLastRequestedBlocks( token ); + String fileName = connections.get( token ).filepath; + long fileSize = new File( fileName ).length(); + long actual; for ( Integer block : blocks ) { - uploader.sendRange( block*Globals.blockSize, (block+1)*Globals.blockSize - 1 ); - uploader.sendFile( connections.get( token ).filepath ); - uploader.close(); + actual = block * Globals.blockSize; + uploader.sendRange( (int)actual, ( ( fileSize - actual ) < Globals.blockSize ) ? (int)fileSize - 1 : ( block + 1 ) * Globals.blockSize - 1 ); // TODO: change this int into long + uploader.sendFile( fileName ); } - + uploader.close(); + } /** @@ -147,9 +153,8 @@ public class ConnectionHandler implements IncomingEvent @Override public void incomingDownloader( Downloader downloader ) throws IOException { + log.debug( "Client wants to upload" ); int startOfRange = 0; - int endOfRange = 0; - int diffOfRange = 0; String token = ""; // try to read meta data while ( downloader.readMetaData() ) { @@ -161,10 +166,11 @@ public class ConnectionHandler implements IncomingEvent return; } - // get range startOfRange = downloader.getStartOfRange(); - endOfRange = downloader.getEndOfRange(); - diffOfRange = downloader.getDiffOfRange(); + + if ( downloader.getDiffOfRange() <= 0 ) { + return; + } // check if he was a uploading client if ( connections.get( token ).type == Connection.DOWNLOADING ) { @@ -172,19 +178,15 @@ public class ConnectionHandler implements IncomingEvent downloader.close(); return; } + + int blockNumber = startOfRange / Globals.blockSize; + UploadingImage image = connections.get( token ).image; + image.setNeedsCheck( blockNumber ); + image.increaseTransmittedTimes( blockNumber ); + log.debug( "Block " + blockNumber + " was transmitted " + image.getTimesTransmitted( blockNumber ) + " time(s)." ); downloader.setOutputFilename( connections.get( token ).filepath ); downloader.readBinary(); - - // calculate and register the incoming blocks - if ( diffOfRange == 0 ) - return; - - for ( int i = startOfRange / Globals.blockSize; i < endOfRange / Globals.blockSize; i += Globals.blockSize ) { - UploadingImage image = connections.get( token ).image; - image.setNeedsCheck( i ); - image.increaseTransmittedTimes( i ); - } } downloader.close(); } diff --git a/src/main/java/org/openslx/imagemaster/serverconnection/ImageProcessor.java b/src/main/java/org/openslx/imagemaster/serverconnection/ImageProcessor.java index f7f9faa..a27d543 100644 --- a/src/main/java/org/openslx/imagemaster/serverconnection/ImageProcessor.java +++ b/src/main/java/org/openslx/imagemaster/serverconnection/ImageProcessor.java @@ -1,7 +1,6 @@ package org.openslx.imagemaster.serverconnection; import java.io.IOException; -import java.sql.Timestamp; import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; @@ -69,9 +68,7 @@ public class ImageProcessor throws UploadException, ImageDataException { // check image data - if ( DbImage.exists( imageData.uuid ) ) { - throw new ImageDataException( ImageDataError.INVALID_DATA, "UUID already existing." ); - } else if ( imageData.imageName == null || imageData.imageName.isEmpty() ) { + if ( imageData.imageName == null || imageData.imageName.isEmpty() ) { throw new ImageDataException( ImageDataError.INVALID_DATA, "Image name not set." ); } else if ( imageData.imageName == null || imageData.imageOwner.isEmpty() ) { throw new ImageDataException( ImageDataError.INVALID_DATA, "Image owner not set." ); @@ -83,6 +80,8 @@ public class ImageProcessor throw new ImageDataException( ImageDataError.INVALID_DATA, "User is not known." ); } + log.debug( serverSessionId + " is submitting " + imageData.uuid ); + String uuid = imageData.uuid; String token; String filepath; @@ -95,19 +94,22 @@ public class ImageProcessor if ( ( image = uploadingImages.get( uuid ) ) != null ) { log.debug( "Image is already uploading.. returning some missing blocks" ); List missing = getNMissingBlocks( image, AMOUNT ); - if ( missing.isEmpty() ) { + if ( missing.isEmpty() && image.allBlocksValid() ) { uploadDone( uuid ); - return new UploadInfos( null, 0, missing ); + return new UploadInfos( image.getToken(), Globals.getSslSocketPort(), missing, image.allBlocksValid() ); } - return new UploadInfos( image.getToken(), Globals.getSslSocketPort(), missing ); + return new UploadInfos( image.getToken(), Globals.getSslSocketPort(), missing, false ); } // insert new image + if ( DbImage.exists( imageData.uuid ) ) { + throw new ImageDataException( ImageDataError.INVALID_DATA, "UUID already existing." ); + } if ( !CRCFile.sumsAreValid( crcSums ) ) throw new UploadException( UploadError.INVALID_CRC, "CRC sums were invalid." ); filepath = Globals.getImageDir() + "/" + uuid + ".vmdk"; token = RandomString.generate( 100, false ); - nBlocks = (int)Math.ceil( imageData.fileSize / Globals.blockSize ); + nBlocks = crcSums.size() - 1; allBlocks = new int[ nBlocks ]; // initalize array with all zeros which mean that this block is missing image = new UploadingImage( token, allBlocks, System.currentTimeMillis(), uuid, filepath ); uploadingImages.put( uuid, image ); @@ -126,9 +128,11 @@ public class ImageProcessor ConnectionHandler.addConnection( token, filepath, Connection.UPLOADING ).image = image; DbImage.insert( imageData, System.currentTimeMillis(), token, nBlocks, serverSessionId, filepath ); imagesToCheck.add( uuid ); + log.debug( imagesToCheck ); log.debug( "Returning UploadInfos. Client should goint to start the upload. " ); - return new UploadInfos( token, Globals.getSslSocketPort(), getNMissingBlocks( image, AMOUNT ) ); + log.debug( image.toString() ); + return new UploadInfos( token, Globals.getSslSocketPort(), getNMissingBlocks( image, AMOUNT ), false ); } public static DownloadInfos getDownloadInfos( String serverSessionId, String uuid, List requestedBlocks ) @@ -198,19 +202,22 @@ public class ImageProcessor */ private static List getNMissingBlocks( UploadingImage image, int amount ) throws UploadException { - int size = image.getAmountOfMissingBlocks(); - if ( amount > size ) - amount = size; + int missing = image.getAmountOfBlocksNeedingRequest(); + log.debug( "The number of missing blocks: " + missing ); + if ( amount > missing ) + amount = missing; List result = new ArrayList<>( amount ); int got = 0; for ( int i = 0; i < image.getNumberOfBlocks(); i++ ) { + log.debug( i ); if ( image.needsRequest( i ) ) { int times = image.getTimesTransmitted( i ); - if (times > 20) { // TODO: make configurable - throw new UploadException(UploadError.BROKEN_BLOCK, "Block " + i + " was transmitted " - + times + " and is still not valid."); + if ( times > 20 ) { // TODO: make configurable + log.debug( "Block " + i + " is probably broken." ); + throw new UploadException( UploadError.BROKEN_BLOCK, "Block " + i + " was transmitted " + + times + " and is still not valid." ); } result.add( i ); got++; @@ -219,6 +226,8 @@ public class ImageProcessor break; } + log.debug( "Returned " + got + " missing blocks." ); + return result; } @@ -232,6 +241,7 @@ public class ImageProcessor { synchronized ( imagesToCheck ) { imagesToCheck.remove( uuid ); + log.debug( "Removing " + uuid ); } UploadingImage image; @@ -248,8 +258,10 @@ public class ImageProcessor { List result = new LinkedList<>(); Iterator iter = imagesToCheck.iterator(); + log.debug( imagesToCheck ); while ( iter.hasNext() ) { result.add( uploadingImages.get( iter.next() ) ); + log.debug( "Added an image ... " ); } return result; } @@ -268,9 +280,9 @@ public class ImageProcessor List list = DbImage.getUploadingImages(); for ( DbImage image : list ) { String token = image.token; - ConnectionHandler.addConnection( token, image.imagePath, Connection.UPLOADING ); UploadingImage infos = new UploadingImage( token, image.blockStatus, image.timestamp.getTime(), image.uuid, image.imagePath ); + ConnectionHandler.addConnection( token, image.imagePath, Connection.UPLOADING ).image = infos; CRCFile crcFile = new CRCFile( Globals.getImageDir() + "/" + image.uuid + ".crc" ); try { if ( !crcFile.isValid() ) @@ -280,6 +292,7 @@ public class ImageProcessor } infos.setCrcFile( crcFile ); uploadingImages.put( image.uuid, infos ); + imagesToCheck.add( image.uuid ); } log.info( "Added " + list.size() + " pending upload(s) to process list again." ); } diff --git a/src/main/java/org/openslx/imagemaster/serverconnection/UploadingImage.java b/src/main/java/org/openslx/imagemaster/serverconnection/UploadingImage.java index dc15dae..8cbaecf 100644 --- a/src/main/java/org/openslx/imagemaster/serverconnection/UploadingImage.java +++ b/src/main/java/org/openslx/imagemaster/serverconnection/UploadingImage.java @@ -1,5 +1,8 @@ package org.openslx.imagemaster.serverconnection; +import java.util.ArrayList; +import java.util.List; + import org.apache.log4j.Logger; import org.openslx.imagemaster.Globals; import org.openslx.imagemaster.crcchecker.CRCFile; @@ -40,6 +43,7 @@ public class UploadingImage this.timestamp = timestamp; this.uuid = uuid; this.blockStatus = initialBlockStatus; + this.filename = filename; } protected void setValid( int index ) @@ -48,6 +52,21 @@ public class UploadingImage return; blockStatus[index] = valid; } + + protected void updateDb() + { + if ( blockStatus == null ) + return; + + List missingBlocks = new ArrayList<>(); + + for (int block = 0; block < blockStatus.length; block++) { + if (blockStatus[block] != valid) { + missingBlocks.add( block ); + } + } + dbImage.updateMissingBlocks( missingBlocks ); + } protected void setMissing( int index ) { @@ -72,9 +91,9 @@ public class UploadingImage protected void increaseTransmittedTimes( int index ) { - if ( blockStatus == null || blockStatus[index] == 0 || blockStatus[index] == 200 ) + if ( blockStatus == null || blockStatus[index] == 200 ) return; - blockStatus[index] += ( blockStatus[index] < missing ) ? -1 : 1; // increase in both directions + blockStatus[index] += ( blockStatus[index] <= missing ) ? -1 : 1; // increase in both directions } protected int getTimesTransmitted( int index ) @@ -91,7 +110,7 @@ public class UploadingImage { if ( blockStatus == null ) return false; - return ( blockStatus[index] > missing && blockStatus[index] != valid ); + return ( ( blockStatus[index] >= missing ) && ( blockStatus[index] != valid ) ); } protected boolean needsCheck( int index ) @@ -103,6 +122,13 @@ public class UploadingImage protected int getNumberOfBlocks() { + /////////////////////////////////////////////////////////////////// + ArrayList l = new ArrayList( blockStatus.length ); + for ( int i : blockStatus ) { + l.add( i ); + } + log.debug( l ); + /////////////////////////////////////////////////////////////////// return blockStatus.length; } @@ -137,15 +163,34 @@ public class UploadingImage this.crcFile = crcFile; } - public int getAmountOfMissingBlocks() + public int getAmountOfBlocksNeedingRequest() { if ( blockStatus == null ) return 0; int result = 0; - for ( int i : blockStatus ) { - if ( blockStatus[i] == missing ) + for ( int i = 0; i < blockStatus.length; i++ ) { + if ( needsRequest( i ) ) result++; } return result; } + + public boolean allBlocksValid() + { + if ( blockStatus == null ) + return false; + for ( int i : blockStatus ) { + if ( i != 200 ) + return false; + } + return true; + } + + @Override + public String toString() + { + return "UUID: " + uuid + ", filename " + filename + "\nmissing blocks " + getAmountOfBlocksNeedingRequest() + + ", number of blocks " + getNumberOfBlocks() + ", token " + getToken(); + + } } diff --git a/src/main/properties/log4j.properties b/src/main/properties/log4j.properties index ced1f8d..1a15589 100644 --- a/src/main/properties/log4j.properties +++ b/src/main/properties/log4j.properties @@ -4,8 +4,9 @@ log4j.appender.A1=org.apache.log4j.ConsoleAppender log4j.appender.A1.layout=org.apache.log4j.PatternLayout -log4j.appender.A1.layout.ConversionPattern=%d [%t] %-5p %c - %m%n +log4j.appender.A1.layout.ConversionPattern=%d [%t] %-5p %F - %m%n log4j.logger.org.apache.directory=ERROR log4j.logger.org.apache.mina=ERROR -log4j.logger.org.apache.ftpserver=ERROR \ No newline at end of file +log4j.logger.org.apache.ftpserver=ERROR +log4j.logger.org.openslx.filetransfer=ERROR \ No newline at end of file -- cgit v1.2.3-55-g7522