From 0e2d23f82f8c564b7267c14414e38de9c71a081f Mon Sep 17 00:00:00 2001 From: Nils Schwabe Date: Fri, 11 Jul 2014 15:46:18 +0200 Subject: Change now saving status of blocks in array --- .../java/org/openslx/imagemaster/db/DbImage.java | 22 ++--- .../imagemaster/serverconnection/CRCScheduler.java | 29 ++++--- .../serverconnection/ConnectionHandler.java | 2 +- .../serverconnection/ImageProcessor.java | 31 +++---- .../serverconnection/UploadingImage.java | 98 ++++++++++++++-------- 5 files changed, 107 insertions(+), 75 deletions(-) diff --git a/src/main/java/org/openslx/imagemaster/db/DbImage.java b/src/main/java/org/openslx/imagemaster/db/DbImage.java index fa089d4..904442d 100644 --- a/src/main/java/org/openslx/imagemaster/db/DbImage.java +++ b/src/main/java/org/openslx/imagemaster/db/DbImage.java @@ -3,9 +3,10 @@ package org.openslx.imagemaster.db; import java.sql.Timestamp; import java.text.SimpleDateFormat; import java.util.Date; -import java.util.LinkedList; import java.util.List; +import org.openslx.imagemaster.Globals; +import org.openslx.imagemaster.serverconnection.UploadingImage; import org.openslx.imagemaster.thrift.iface.ImageData; public class DbImage @@ -26,7 +27,7 @@ public class DbImage public final Timestamp timestamp; public final long fileSize; public final String token; - public final List missingBlocks; + public final int[] blockStatus; public final String serverSessionId; public DbImage(String uuid) @@ -46,7 +47,7 @@ public class DbImage this.timestamp = new Timestamp( 0 ); this.fileSize = 0; this.token = null; - this.missingBlocks = null; + this.blockStatus = null; this.serverSessionId = null; } @@ -72,9 +73,12 @@ public class DbImage this.token = token; String[] parts = missingBlocksList.split( ";" ); - this.missingBlocks = new LinkedList<>(); + blockStatus = new int[ (int)Math.ceil( this.fileSize / Globals.blockSize ) ]; // initialize array to ones + for ( int i : blockStatus ) { + blockStatus[i] = UploadingImage.valid; + } for ( int i = 0; i < parts.length - 1; i++ ) { // do not copy the last empty string (1;2;3;) -> "1","2","3","" - this.missingBlocks.add( Integer.valueOf( parts[i] ) ); + blockStatus[i] = UploadingImage.missing; } this.serverSessionId = serverSessionId; @@ -101,7 +105,7 @@ public class DbImage * @param serverSessionId The server that is uploading this image * @return Affected rows */ - public static int insert( ImageData imageData, long ts, String token, List missingBlocks, String serverSessionId, String filepath ) + public static int insert( ImageData imageData, long ts, String token, int missingBlocks, String serverSessionId, String filepath ) { Date createTime = new Date( imageData.imageCreateTime ); Date updateTime = new Date( imageData.imageUpdateTime ); @@ -109,10 +113,8 @@ public class DbImage SimpleDateFormat sdf = new SimpleDateFormat( "yyyy-MM-dd HH:mm:ss" ); String missingBlocksList = ""; - if ( missingBlocks != null ) { - for ( Integer block : missingBlocks ) { - missingBlocksList = missingBlocksList + String.valueOf( block ) + ";"; - } + for ( int i = 0; i < missingBlocks; i++ ) { + missingBlocksList = missingBlocksList + String.valueOf( i ) + ";"; } return MySQL diff --git a/src/main/java/org/openslx/imagemaster/serverconnection/CRCScheduler.java b/src/main/java/org/openslx/imagemaster/serverconnection/CRCScheduler.java index a817738..065ca14 100644 --- a/src/main/java/org/openslx/imagemaster/serverconnection/CRCScheduler.java +++ b/src/main/java/org/openslx/imagemaster/serverconnection/CRCScheduler.java @@ -2,7 +2,6 @@ package org.openslx.imagemaster.serverconnection; import java.io.IOException; import java.util.Iterator; -import java.util.LinkedList; import java.util.List; import java.util.Timer; import java.util.TimerTask; @@ -10,7 +9,7 @@ import java.util.TimerTask; import org.openslx.imagemaster.crcchecker.CRCChecker; /** - * Class to schedule crc checks. + * Class to schedule crc checks. */ public class CRCScheduler extends TimerTask { @@ -22,23 +21,29 @@ public class CRCScheduler extends TimerTask Iterator iter = list.iterator(); while ( iter.hasNext() ) { UploadingImage image = iter.next(); - List blocks = image.getNotCheckedBlocks(); - List finishedBlocks = new LinkedList<>(); - try { - finishedBlocks = CRCChecker.checkCRC( image.getFilename(), image.getCrcFilename(), blocks ); - } catch ( IOException e ) { - // TODO: Could not read crc file: tell this to client + CRCChecker crcChecker = new CRCChecker( image.getFilename(), image.getCrcFilename() ); + for ( int block = 0; block < image.getNumberOfBlocks(); block++ ) { + if ( image.needsCheck( block ) ) { + try { + if ( crcChecker.checkBlock( block ) ) { + image.setValid( block ); + } else { + image.setNeedsRequest( block ); + } + } catch ( IOException e ) { + // TODO: Handle that crc file or image file could not be read. + } + } } - image.removeBlocks( finishedBlocks ); } } - + public static void startScheduling() { Timer timer = new Timer( "CRCScheduler" ); - + // start now and fire every 60 s timer.schedule( new CRCScheduler(), 0, 60000 ); } - + } diff --git a/src/main/java/org/openslx/imagemaster/serverconnection/ConnectionHandler.java b/src/main/java/org/openslx/imagemaster/serverconnection/ConnectionHandler.java index 7a3bf12..c273239 100644 --- a/src/main/java/org/openslx/imagemaster/serverconnection/ConnectionHandler.java +++ b/src/main/java/org/openslx/imagemaster/serverconnection/ConnectionHandler.java @@ -186,7 +186,7 @@ public class ConnectionHandler implements IncomingEvent return; for ( int i = startOfRange / Globals.blockSize; i < endOfRange / Globals.blockSize; i += Globals.blockSize ) { - connections.get( token ).image.addBlockToCheck( i ); + connections.get( token ).image.setNeedsCheck( i ); } } diff --git a/src/main/java/org/openslx/imagemaster/serverconnection/ImageProcessor.java b/src/main/java/org/openslx/imagemaster/serverconnection/ImageProcessor.java index 10580a7..8e7c011 100644 --- a/src/main/java/org/openslx/imagemaster/serverconnection/ImageProcessor.java +++ b/src/main/java/org/openslx/imagemaster/serverconnection/ImageProcessor.java @@ -68,8 +68,8 @@ public class ImageProcessor String token; String filepath; String crcPath; - - List allBlocks; + int nBlocks; + int[] allBlocks; UploadingImage image; synchronized ( uploadingImages ) { @@ -88,24 +88,21 @@ public class ImageProcessor filepath = Globals.getImageDir() + "/" + uuid + ".vmdk"; token = RandomString.generate( 100, false ); crcPath = Globals.getImageDir() + "/" + uuid + ".crc"; - int nBlocks = (int)Math.ceil( imageData.fileSize / Globals.blockSize ); - allBlocks = new ArrayList<>( nBlocks ); - for ( int i = nBlocks - 1; i >= 0; i-- ) { // fill empty list with all block numbers - allBlocks.add( i ); - } + nBlocks = (int)Math.ceil( imageData.fileSize / Globals.blockSize ); + allBlocks = new int[nBlocks]; // initalize array with all zeros which mean that this block is missing image = new UploadingImage( token, allBlocks, new Timestamp( System.currentTimeMillis() ), uuid, filepath, crcPath ); uploadingImages.put( uuid, image ); } try { - new CRCFile( crcSums, crcPath); + CRCFile.writeCrcFile( crcSums, crcPath ); } catch (IOException e) { log.error( "Could not create crc file", e ); return null; // TODO: what to do if we can not write the crc file to disk? Give object to crcscheduler? } ConnectionHandler.addConnection( token, filepath, Connection.UPLOADING ).image = image; - DbImage.insert( imageData, System.currentTimeMillis(), token, allBlocks, serverSessionId, filepath ); + DbImage.insert( imageData, System.currentTimeMillis(), token, nBlocks, serverSessionId, filepath ); imagesToCheck.add( uuid ); log.debug( "Returning UploadInfos. Client should goint to start the upload. " ); @@ -178,14 +175,19 @@ public class ImageProcessor */ private static List getNMissingBlocks( UploadingImage image, int amount ) { - int size = image.amountOfMissingBlocks(); + int size = image.getAmountOfMissingBlocks(); if ( amount > size ) amount = size; List result = new ArrayList<>( amount ); - for ( int i = 1; i <= amount; i++ ) { - result.add( image.removeMissingBlock( size - i ) ); + int got = 0; + for ( int i = 0; i < image.getNumberOfBlocks(); i++ ) { + if (image.needsRequest( i )) { + result.add( i ) ; + got++; + } + if (got == amount) break; } return result; @@ -220,12 +222,11 @@ public class ImageProcessor while ( iter.hasNext() ) { result.add( uploadingImages.get( iter.next() ) ); } - return result; } public static List getRequestedBlocks( String token) { - + // for the downloader return null; } @@ -238,7 +239,7 @@ public class ImageProcessor for ( DbImage image : list ) { String token = image.token; ConnectionHandler.addConnection( token, image.imagePath, Connection.UPLOADING ); - UploadingImage infos = new UploadingImage( token, image.missingBlocks, image.timestamp, + UploadingImage infos = new UploadingImage( token, image.blockStatus, image.timestamp, image.uuid, image.imagePath, Globals.getImageDir() + "/" + image.uuid + ".crc" ); uploadingImages.put( image.uuid, infos ); } diff --git a/src/main/java/org/openslx/imagemaster/serverconnection/UploadingImage.java b/src/main/java/org/openslx/imagemaster/serverconnection/UploadingImage.java index 0e36175..27e18fd 100644 --- a/src/main/java/org/openslx/imagemaster/serverconnection/UploadingImage.java +++ b/src/main/java/org/openslx/imagemaster/serverconnection/UploadingImage.java @@ -1,9 +1,6 @@ package org.openslx.imagemaster.serverconnection; import java.sql.Timestamp; -import java.util.Collection; -import java.util.LinkedList; -import java.util.List; import org.apache.log4j.Logger; import org.openslx.imagemaster.db.DbImage; @@ -13,57 +10,70 @@ import org.openslx.imagemaster.db.DbImage; */ public class UploadingImage { + public static final Logger log = Logger.getLogger( UploadingImage.class ); /** * Token for the satellite. */ private String token; /** - * The missing blocks that need to be uploaded by the satellite. - */ - private List missingBlocks; - /** - * The list of blocks that were requested but not checked + * The status list of the blocks. + * x = 0 block is missing + * x = 200 block arrived and is valid + * x > 0 block is invalid and was transmitted x times (needs request) + * x < 0 block is invalid and was transmitted x times (needs check) */ - private List blocksToCheck = new LinkedList<>(); + private int[] blockStatus = null; + public static final int valid = 200; + public static final int missing = 0; private Timestamp ts; // when did the server something for the last time private DbImage dbImage = null; // the DB representation of this image private String uuid; private String filename; private String crcFilename; - protected UploadingImage(String token, List missingBlocks, Timestamp ts, String uuid, String filename, String crcFilename) + protected UploadingImage(String token, int[] initialBlockStatus, Timestamp ts, String uuid, String filename, String crcFilename) { this.token = token; - this.missingBlocks = missingBlocks; this.ts = ts; this.uuid = uuid; this.crcFilename = crcFilename; + this.blockStatus = initialBlockStatus; } - protected void removeBlock( int number ) + protected void setValid( int index ) { - this.missingBlocks.remove( number ); - dbImage.updateMissingBlocks( missingBlocks ); + if ( blockStatus == null ) + return; + blockStatus[index] = valid; } - protected void removeBlocks( Collection list ) + protected void setMissing( int index ) { - this.missingBlocks.removeAll( list ); - dbImage.updateMissingBlocks( missingBlocks ); + if ( blockStatus == null ) + return; + blockStatus[index] = missing; } - - protected void addBlockToCheck( int number ) + + protected void setNeedsRequest( int index ) { - synchronized ( blocksToCheck ) { - blocksToCheck.add( number ); - } - log.debug( number + " added to check list..." ); + if ( blockStatus == null ) + return; + blockStatus[index] *= ( blockStatus[index] < missing ) ? -1 : 1; // switch to positive value if needed + } + + protected void setNeedsCheck( int index ) + { + if ( blockStatus == null ) + return; + blockStatus[index] *= ( blockStatus[index] > missing ) ? -1 : 1; // switch to negative value if needed } - protected List getNotCheckedBlocks() + protected void increaseTransmittedTimes( int index ) { - return this.blocksToCheck; + if ( blockStatus == null || blockStatus[index] == 0 || blockStatus[index] == 200 ) + return; + blockStatus[index] += ( blockStatus[index] < missing ) ? -1 : 1; // increase in both directions } protected String getToken() @@ -71,9 +81,23 @@ public class UploadingImage return this.token; } - protected List getAllMissingBlocks() + protected boolean needsRequest( int index ) + { + if ( blockStatus == null ) + return false; + return ( blockStatus[index] > missing && blockStatus[index] != valid ); + } + + protected boolean needsCheck( int index ) + { + if ( blockStatus == null ) + return false; + return ( blockStatus[index] < missing ); + } + + protected int getNumberOfBlocks() { - return this.missingBlocks; + return blockStatus.length; } protected Timestamp getTimestamp() @@ -88,26 +112,26 @@ public class UploadingImage } return this.dbImage; } - + protected String getFilename() { return this.filename; } - + protected String getCrcFilename() { return this.crcFilename; } - public int amountOfMissingBlocks() - { - return missingBlocks.size(); - } - - public int removeMissingBlock( int index ) + public int getAmountOfMissingBlocks() { - synchronized ( missingBlocks ) { - return missingBlocks.remove( index ); + if ( blockStatus == null ) + return 0; + int result = 0; + for ( int i : blockStatus ) { + if ( blockStatus[i] == missing ) + result++; } + return result; } } -- cgit v1.2.3-55-g7522