summaryrefslogtreecommitdiffstats
path: root/src/main/java/org/openslx/imagemaster/serverconnection/ImageProcessor.java
diff options
context:
space:
mode:
Diffstat (limited to 'src/main/java/org/openslx/imagemaster/serverconnection/ImageProcessor.java')
-rw-r--r--src/main/java/org/openslx/imagemaster/serverconnection/ImageProcessor.java297
1 files changed, 58 insertions, 239 deletions
diff --git a/src/main/java/org/openslx/imagemaster/serverconnection/ImageProcessor.java b/src/main/java/org/openslx/imagemaster/serverconnection/ImageProcessor.java
index d5e253c..739bc62 100644
--- a/src/main/java/org/openslx/imagemaster/serverconnection/ImageProcessor.java
+++ b/src/main/java/org/openslx/imagemaster/serverconnection/ImageProcessor.java
@@ -1,9 +1,7 @@
package org.openslx.imagemaster.serverconnection;
import java.io.File;
-import java.io.IOException;
import java.util.ArrayList;
-import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
@@ -15,13 +13,13 @@ import org.openslx.imagemaster.Globals;
import org.openslx.imagemaster.crcchecker.CrcFile;
import org.openslx.imagemaster.db.DbImage;
import org.openslx.imagemaster.db.DbUser;
-import org.openslx.imagemaster.thrift.iface.DownloadInfos;
+import org.openslx.imagemaster.thrift.iface.DownloadData;
import org.openslx.imagemaster.thrift.iface.ImageData;
import org.openslx.imagemaster.thrift.iface.ImageDataError;
import org.openslx.imagemaster.thrift.iface.ImageDataException;
+import org.openslx.imagemaster.thrift.iface.UploadData;
import org.openslx.imagemaster.thrift.iface.UploadError;
import org.openslx.imagemaster.thrift.iface.UploadException;
-import org.openslx.imagemaster.thrift.iface.UploadInfos;
import org.openslx.imagemaster.util.RandomString;
import org.openslx.imagemaster.util.Util;
@@ -35,10 +33,6 @@ public class ImageProcessor
private static final Logger log = Logger.getLogger( ImageProcessor.class );
/**
- * The amount of blocks that is return in UploadInfos (after request of satellite)
- */
- private static final int AMOUNT = 20;
- /**
* The uploading images.
* Key: imageUUID,
* Value: uploadingImageInfos
@@ -48,14 +42,7 @@ public class ImageProcessor
/**
* The UUIDs of the images that need to be checked by the crc checker.
*/
- private static List<String> imagesToCheck = new LinkedList<>();
-
- /**
- * The downloading clients.
- * Key: serverSessionId
- * Value: downloadingClientInfos
- */
- private static HashMap<String, DownloadingClient> downloadingClients = new HashMap<>();
+ private static List<String> imagesToCheck = new ArrayList<>();
/**
* Checks if this image is already uploading and returns a new list with missing blocks if so.
@@ -66,218 +53,81 @@ public class ImageProcessor
* @return
* @throws UploadException If some error occurred during the process
*/
- public static UploadInfos getUploadInfos( String serverSessionId, ImageData imageData, List<Integer> crcSums )
+ public static UploadData getUploadInfos( ImageData imageData, List<Integer> crcSums )
throws UploadException, ImageDataException
{
// check image data
- if ( imageData.imageName == null || imageData.imageName.isEmpty() ) {
+ if ( imageData.imageName == null || imageData.imageName.isEmpty() )
throw new ImageDataException( ImageDataError.INVALID_DATA, "Image name not set." );
- } else if ( imageData.imageOwner == null || imageData.imageOwner.isEmpty() ) {
- throw new ImageDataException( ImageDataError.INVALID_DATA, "Image owner not set." );
- } else if ( imageData.contentOperatingSystem == null || imageData.contentOperatingSystem.isEmpty() ) {
+ if ( imageData.imageOwner == null || !DbUser.exists( imageData.imageOwner ) )
+ throw new ImageDataException( ImageDataError.INVALID_DATA, "Invalid image owner." );
+ if ( imageData.contentOperatingSystem == null || imageData.contentOperatingSystem.isEmpty() )
throw new ImageDataException( ImageDataError.INVALID_DATA, "Content operating system not set." );
- } else if ( imageData.fileSize <= 0 ) {
+ if ( imageData.fileSize <= 0 )
throw new ImageDataException( ImageDataError.INVALID_DATA, "File size is too small." );
- } else if ( !DbUser.exists( imageData.imageOwner ) ) {
- throw new ImageDataException( ImageDataError.INVALID_DATA, "User is not known." );
- }
-
- //TODO: this is not working like this:
- DbImage i = DbImage.getImageByUuid( imageData.uuid );
-// boolean isUpdate = false;
-// if ( i != null ) {
-// // image is already available
-// // is the client updating??
-// if ( imageData.imageVersion <= i.imageVersion ) {
-// throw new ImageDataException( ImageDataError.INVALID_DATA, "This image with the same or a newer version is already available." );
-// } else {
-// // TODO: update db and prepare for new image file
-// isUpdate = true;
-// }
-// }
- log.debug( serverSessionId + " is submitting " + imageData.uuid );
+ log.debug( "Satellite is submitting " + imageData.uuid );
- String uuid = imageData.uuid;
- String token;
- String filepath;
- int nBlocks;
- int[] allBlocks;
+ final String uuid = imageData.uuid;
+ final String filepathRelative;
+ final CrcFile crcFile = new CrcFile( crcSums );
UploadingImage image;
synchronized ( uploadingImages ) {
// check if image is already uploading
- if ( ( image = uploadingImages.get( uuid ) ) != null ) {
- if ( image.getCrcFile() == null ) {
- CrcFile crcFile;
- try {
- // try to write crc file ...
- crcFile = CrcFile.writeCrcFile( crcSums, generateFilepathOfCrcFile( imageData ) );
- } catch ( IOException e ) {
- // ... and keep it in ram if it fails
- crcFile = new CrcFile( crcSums );
- }
- image.setCrcFile( crcFile );
+ if ( ( image = uploadingImages.get( uuid ) ) == null ) {
+ // insert new image
+ if ( crcSums != null && !crcFile.isValid() )
+ throw new UploadException( UploadError.INVALID_CRC, "CRC sums were invalid." );
+ filepathRelative = generateFilepathOfImage( imageData );
+ DbImage.insert( imageData, filepathRelative ); // Make sure it exists in DB
+ try {
+ image = new UploadingImage( uuid );
+ } catch ( Exception e ) {
+ throw new UploadException( UploadError.GENERIC_ERROR, "Internal error" );
}
- List<Integer> missing = getNMissingBlocks( image, AMOUNT );
- if ( missing.isEmpty() && image.allBlocksValid() ) {
- uploadDone( uuid );
- return new UploadInfos( image.getToken(), Globals.getSslSocketPort(), missing, image.allBlocksValid() );
- }
- return new UploadInfos( image.getToken(), Globals.getSslSocketPort(), missing, false );
- }
-
- // insert new image
- if ( !CrcFile.sumsAreValid( crcSums ) )
- throw new UploadException( UploadError.INVALID_CRC, "CRC sums were invalid." );
- filepath = generateFilepathOfImage( imageData );
- token = RandomString.generate( 100, false );
- nBlocks = Util.getNumberOfBlocks( imageData.fileSize, Globals.blockSize );
- allBlocks = new int[ nBlocks ]; // initialize array with all zeros which mean that this block is missing
- image = new UploadingImage( token, allBlocks, System.currentTimeMillis(), uuid, filepath );
- image.setDbImage( i ); // set the dbImage (it doesn't matter if the image is null because the uploadingImage is creating it then
- uploadingImages.put( uuid, image );
- }
-
- CrcFile crcFile;
- try {
- // try to write crc file ...
- crcFile = CrcFile.writeCrcFile( crcSums, generateFilepathOfCrcFile( imageData ) );
- } catch ( IOException e ) {
- // ... and keep it in ram if it fails
- crcFile = new CrcFile( crcSums );
- }
- image.setCrcFile( crcFile );
-
- ConnectionHandler.addConnection( token, filepath, Connection.UPLOADING ).image = image;
-// if ( isUpdate ) {
-// i.updateVersion( i.imageVersion, Util.getNumberOfBlocks( i.fileSize, Globals.blockSize ) );
-// } else {
- DbImage.insert( imageData, System.currentTimeMillis(), token, nBlocks, serverSessionId, filepath );
-// }
- imagesToCheck.add( uuid );
- log.debug( imagesToCheck );
-
- log.debug( image.toString() );
- return new UploadInfos( token, Globals.getSslSocketPort(), getNMissingBlocks( image, AMOUNT ), false );
- }
-
- public static DownloadInfos getDownloadInfos( String serverSessionId, String uuid, List<Integer> requestedBlocks )
- {
- // check if server is already downloading
- if ( downloadingClients.containsKey( serverSessionId ) ) {
- DownloadingClient client = downloadingClients.get( serverSessionId );
-
- // remove download if done
- if ( requestedBlocks.isEmpty() )
- {
- downloadDone( serverSessionId, uuid );
- return new DownloadInfos();
- }
-
- if ( client.isDownloading( uuid ) )
- {
- // client was downloading this image
- // update the requested blocks
- client.requestBlocks( uuid, requestedBlocks );
- return new DownloadInfos( client.getToken( uuid ), Globals.getSslSocketPort() );
+ uploadingImages.put( uuid, image );
}
-
- // server was downloading another image and now gets a new connection for this new download
- String token = RandomString.generate( 100, false );
- String filepath = DbImage.getImageByUuid( uuid ).imagePath;
- ConnectionHandler.addConnection( token, filepath, Connection.DOWNLOADING );
-
- client.addDownload( uuid, requestedBlocks, token );
- downloadingClients.put( serverSessionId, client );
- return new DownloadInfos( token, Globals.getSslSocketPort() );
}
- // insert new client and start listener
- synchronized ( downloadingClients ) {
- String token = RandomString.generate( 100, false );
- String filepath = DbImage.getImageByUuid( uuid ).imagePath;
-
- DownloadingClient client = new DownloadingClient();
- client.addDownload( uuid, requestedBlocks, token );
- downloadingClients.put( serverSessionId, client );
+ final String token = RandomString.generate( 50, false );
- ConnectionHandler.addConnection( token, filepath, Connection.DOWNLOADING ).client = client;
- return new DownloadInfos( token, Globals.getSslSocketPort() );
- }
+ ConnectionHandler.addUpload( token, image );
+ // Set crc file on image - if there is already a crc file assigned, this does nothing
+ image.setCrcFile( crcFile );
+ if ( image.allBlocksValid() )
+ removeFinishedUploads();
+ return new UploadData( token, Globals.getSslSocketPort() );
}
- private static void downloadDone( String serverSessionId, String uuid )
+ public static DownloadData getDownloadInfos( String uuid ) throws ImageDataException
{
- synchronized ( downloadingClients ) {
- DownloadingClient client = downloadingClients.get( serverSessionId );
- client.removeDownload( uuid );
- ConnectionHandler.removeConnection( client.getToken( uuid ) );
- if ( !client.hasDownloads() ) {
- downloadingClients.remove( serverSessionId );
- }
- }
+ DbImage image = DbImage.getImageByUuid( uuid );
+ if ( image == null )
+ throw new ImageDataException( ImageDataError.UNKNOWN_IMAGE, "UUID '" + uuid + "' does not map to a known image." );
+ // server was downloading another image and now gets a new connection for this new download
+ String token = RandomString.generate( 50, false );
+ ConnectionHandler.addDownload( token, image );
+ return new DownloadData( token, Globals.getSslSocketPort(), null ); // TODO: Return crc list
}
/**
- * Returns a specified number of missing blocks.
- *
- * @param imageUUID The image of which you want to get the missing blocks from
- * @param amount The amount of blocks that you want to get
- * @return The missing blocks
- * @throws UploadException If a block was transmitted to many times.
+ * Go though list of active uploading images and remove
+ * those that are finished.
*/
- private static List<Integer> getNMissingBlocks( UploadingImage image, int amount ) throws UploadException
+ public static void removeFinishedUploads()
{
- int missing = image.getAmountOfBlocksNeedingRequest();
- log.debug( "The number of missing blocks: " + missing );
- if ( amount > missing )
- amount = missing;
- List<Integer> result = new ArrayList<>( amount );
-
- int got = 0;
- for ( int i = 0; i < image.getNumberOfBlocks(); i++ ) {
- if ( image.needsRequest( i ) ) {
- int times = image.getTimesTransmitted( i );
- if ( times > Globals.getSslTransmitTimes() ) {
- log.debug( "Block " + i + " is probably broken." );
- throw new UploadException( UploadError.BROKEN_BLOCK, "Block " + i + " was transmitted "
- + times + " and is still not valid." );
- }
- result.add( i );
- got++;
+ for (Iterator<UploadingImage> it = uploadingImages.values().iterator(); it.hasNext(); ) {
+ UploadingImage image = it.next();
+ if (image.allBlocksValid()) {
+ synchronized ( imagesToCheck ) {
+ imagesToCheck.remove( image.getUuid() );
+ image.updateMissingBlocks( null );
+ }
+ it.remove();
}
- if ( got == amount )
- break;
- }
-
- log.debug( "Returned " + got + " missing blocks." );
-
- return result;
- }
-
- /**
- * Is triggered when an upload of an image is done.
- * Removes image from process list, updates db entry and moves file on hard drive.
- *
- * @param uuid
- */
- private static void uploadDone( String uuid )
- {
- synchronized ( imagesToCheck ) {
- imagesToCheck.remove( uuid );
- log.debug( "Removing " + uuid );
- }
-
- UploadingImage image;
- synchronized ( uploadingImages ) {
- image = uploadingImages.remove( uuid );
}
- image.getDbImage().updateMissingBlocks( null );
- // file was already downloaded in the right location by the updownloader class.
- // remove the connection so that it can be used by a new client
- ConnectionHandler.removeConnection( image.getToken() );
}
public static List<UploadingImage> getImagesToCheck()
@@ -297,28 +147,28 @@ public class ImageProcessor
// TODO
return null;
}
-
+
/**
* Generates the filePath of an image.
* And creates the folder if wanted.
* The crc file is found under filePath + ".crc"
*
* @param imageData The data of the image
- * @param createFolder If you want the folder to be created
* @return The filePath of the given image
*/
public static String generateFilepathOfImage( ImageData imageData )
{
return generateFilePathOfImage( imageData.uuid, imageData.imageName, imageData.imageVersion );
}
-
- public static String generateFilePathOfImage( String uuid, String imageName, int imageVersion) {
- String result = Globals.getImageDir() + "/" + uuid + "/";
- File dir = new File(result);
+
+ public static String generateFilePathOfImage( String uuid, String imageName, int imageVersion )
+ {
+ String result = Util.sanitizeFileName( uuid ) + "/";
+ File dir = new File( Globals.getImageDir() + "/" + result );
if ( !dir.exists() ) {
dir.mkdirs();
}
- result += imageName + "-v" + String.valueOf( imageVersion ) + ".vmdk";
+ result += imageName + "-rev" + String.valueOf( imageVersion );
return result;
}
@@ -326,41 +176,10 @@ public class ImageProcessor
{
return generateFilepathOfImage( imageData ) + ".crc";
}
-
+
public static String generateFilepathOfCrcFile( String uuid, String imageName, int imageVersion )
{
return generateFilePathOfImage( uuid, imageName, imageVersion ) + ".crc";
}
- /**
- * Checks pending uploads in database and adds them to process list again.
- */
- static
- {
- List<DbImage> list = DbImage.getUploadingImages();
- for ( DbImage image : list ) {
- UploadingImage infos = new UploadingImage( image.token, image.blockStatus, image.timestamp.getTime(), image.uuid, image.imagePath );
- ConnectionHandler.addConnection( image.token, image.imagePath, Connection.UPLOADING ).image = infos;
- CrcFile crcFile = new CrcFile( generateFilepathOfCrcFile( image.uuid, image.imageName, image.imageVersion ) ); // TODO: has to be adjusted with the corresponding value above
- try {
- if ( !crcFile.isValid() ) {
- continue;
- // UploadingImage object will contain a CRCFile = null which invokes the ImageProcessor to retry to save it
- }
- } catch ( IOException e ) {
- continue;
- // same thing here
- }
- infos.setCrcFile( crcFile );
- uploadingImages.put( image.uuid, infos );
- imagesToCheck.add( image.uuid );
- }
- log.info( "Added " + list.size() + " pending upload(s) to process list again." );
- }
-
-
-
- public static void init()
- {
- }
}
generated by cgit v1.2.3-55-g7522 (git 2.39.0) at 2024-05-05 16:27:46 +0200