package org.openslx.imagemaster.serverconnection;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import org.apache.log4j.Logger;
import org.openslx.imagemaster.Globals;
import org.openslx.imagemaster.crcchecker.CrcFile;
import org.openslx.imagemaster.db.DbImage;
import org.openslx.imagemaster.db.DbUser;
import org.openslx.imagemaster.thrift.iface.DownloadInfos;
import org.openslx.imagemaster.thrift.iface.ImageData;
import org.openslx.imagemaster.thrift.iface.ImageDataError;
import org.openslx.imagemaster.thrift.iface.ImageDataException;
import org.openslx.imagemaster.thrift.iface.UploadError;
import org.openslx.imagemaster.thrift.iface.UploadException;
import org.openslx.imagemaster.thrift.iface.UploadInfos;
import org.openslx.imagemaster.util.RandomString;
import org.openslx.imagemaster.util.Util;
/**
* Processing the up- and download of images.
* Handles who is authorized and knows which blocks are missing / need to be sent
*/
public class ImageProcessor
{
private static final Logger log = Logger.getLogger( ImageProcessor.class );
/**
* The amount of blocks that is return in UploadInfos (after request of satellite)
*/
private static final int AMOUNT = 20;
/**
* The uploading images.
* Key: imageUUID,
* Value: uploadingImageInfos
*/
private static Map<String, UploadingImage> uploadingImages = new ConcurrentHashMap<>();
/**
* The UUIDs of the images that need to be checked by the crc checker.
*/
private static List<String> imagesToCheck = new LinkedList<>();
/**
* The downloading clients.
* Key: serverSessionId
* Value: downloadingClientInfos
*/
private static HashMap<String, DownloadingClient> downloadingClients = new HashMap<>();
/**
* Checks if this image is already uploading and returns a new list with missing blocks if so.
* Puts the new image into processing list else.
*
* @param serverSessionId The uploading server
* @param imageData The data of the image
* @return
* @throws UploadException If some error occurred during the process
*/
public static UploadInfos getUploadInfos( String serverSessionId, ImageData imageData, List<Integer> crcSums )
throws UploadException, ImageDataException
{
// check image data
if ( imageData.imageName == null || imageData.imageName.isEmpty() ) {
throw new ImageDataException( ImageDataError.INVALID_DATA, "Image name not set." );
} else if ( imageData.imageOwner == null || imageData.imageOwner.isEmpty() ) {
throw new ImageDataException( ImageDataError.INVALID_DATA, "Image owner not set." );
} else if ( imageData.contentOperatingSystem == null || imageData.contentOperatingSystem.isEmpty() ) {
throw new ImageDataException( ImageDataError.INVALID_DATA, "Content operating system not set." );
} else if ( imageData.fileSize <= 0 ) {
throw new ImageDataException( ImageDataError.INVALID_DATA, "File size is too small." );
} else if ( !DbUser.exists( imageData.imageOwner ) ) {
throw new ImageDataException( ImageDataError.INVALID_DATA, "User is not known." );
}
//TODO: this is not working like this:
DbImage i = DbImage.getImageByUuid( imageData.uuid );
// boolean isUpdate = false;
// if ( i != null ) {
// // image is already available
// // is the client updating??
// if ( imageData.imageVersion <= i.imageVersion ) {
// throw new ImageDataException( ImageDataError.INVALID_DATA, "This image with the same or a newer version is already available." );
// } else {
// // TODO: update db and prepare for new image file
// isUpdate = true;
// }
// }
log.debug( serverSessionId + " is submitting " + imageData.uuid );
String uuid = imageData.uuid;
String token;
String filepath;
int nBlocks;
int[] allBlocks;
UploadingImage image;
synchronized ( uploadingImages ) {
// check if image is already uploading
if ( ( image = uploadingImages.get( uuid ) ) != null ) {
if ( image.getCrcFile() == null ) {
CrcFile crcFile;
try {
// try to write crc file ...
crcFile = CrcFile.writeCrcFile( crcSums, generateFilepathOfCrcFile( imageData ) );
} catch ( IOException e ) {
// ... and keep it in ram if it fails
crcFile = new CrcFile( crcSums );
}
image.setCrcFile( crcFile );
}
List<Integer> missing = getNMissingBlocks( image, AMOUNT );
if ( missing.isEmpty() && image.allBlocksValid() ) {
uploadDone( uuid );
return new UploadInfos( image.getToken(), Globals.getSslSocketPort(), missing, image.allBlocksValid() );
}
return new UploadInfos( image.getToken(), Globals.getSslSocketPort(), missing, false );
}
// insert new image
if ( !CrcFile.sumsAreValid( crcSums ) )
throw new UploadException( UploadError.INVALID_CRC, "CRC sums were invalid." );
filepath = generateFilepathOfImage( imageData );
token = RandomString.generate( 100, false );
nBlocks = Util.getNumberOfBlocks( imageData.fileSize, Globals.blockSize );
allBlocks = new int[ nBlocks ]; // initialize array with all zeros which mean that this block is missing
image = new UploadingImage( token, allBlocks, System.currentTimeMillis(), uuid, filepath );
image.setDbImage( i ); // set the dbImage (it doesn't matter if the image is null because the uploadingImage is creating it then
uploadingImages.put( uuid, image );
}
CrcFile crcFile;
try {
// try to write crc file ...
crcFile = CrcFile.writeCrcFile( crcSums, generateFilepathOfCrcFile( imageData ) );
} catch ( IOException e ) {
// ... and keep it in ram if it fails
crcFile = new CrcFile( crcSums );
}
image.setCrcFile( crcFile );
ConnectionHandler.addConnection( token, filepath, Connection.UPLOADING ).image = image;
// if ( isUpdate ) {
// i.updateVersion( i.imageVersion, Util.getNumberOfBlocks( i.fileSize, Globals.blockSize ) );
// } else {
DbImage.insert( imageData, System.currentTimeMillis(), token, nBlocks, serverSessionId, filepath );
// }
imagesToCheck.add( uuid );
log.debug( imagesToCheck );
log.debug( image.toString() );
return new UploadInfos( token, Globals.getSslSocketPort(), getNMissingBlocks( image, AMOUNT ), false );
}
public static DownloadInfos getDownloadInfos( String serverSessionId, String uuid, List<Integer> requestedBlocks )
{
// check if server is already downloading
if ( downloadingClients.containsKey( serverSessionId ) ) {
DownloadingClient client = downloadingClients.get( serverSessionId );
// remove download if done
if ( requestedBlocks.isEmpty() )
{
downloadDone( serverSessionId, uuid );
return new DownloadInfos();
}
if ( client.isDownloading( uuid ) )
{
// client was downloading this image
// update the requested blocks
client.requestBlocks( uuid, requestedBlocks );
return new DownloadInfos( client.getToken( uuid ), Globals.getSslSocketPort() );
}
// server was downloading another image and now gets a new connection for this new download
String token = RandomString.generate( 100, false );
String filepath = DbImage.getImageByUuid( uuid ).imagePath;
ConnectionHandler.addConnection( token, filepath, Connection.DOWNLOADING );
client.addDownload( uuid, requestedBlocks, token );
downloadingClients.put( serverSessionId, client );
return new DownloadInfos( token, Globals.getSslSocketPort() );
}
// insert new client and start listener
synchronized ( downloadingClients ) {
String token = RandomString.generate( 100, false );
String filepath = DbImage.getImageByUuid( uuid ).imagePath;
DownloadingClient client = new DownloadingClient();
client.addDownload( uuid, requestedBlocks, token );
downloadingClients.put( serverSessionId, client );
ConnectionHandler.addConnection( token, filepath, Connection.DOWNLOADING ).client = client;
return new DownloadInfos( token, Globals.getSslSocketPort() );
}
}
private static void downloadDone( String serverSessionId, String uuid )
{
synchronized ( downloadingClients ) {
DownloadingClient client = downloadingClients.get( serverSessionId );
client.removeDownload( uuid );
ConnectionHandler.removeConnection( client.getToken( uuid ) );
if ( !client.hasDownloads() ) {
downloadingClients.remove( serverSessionId );
}
}
}
/**
* Returns a specified number of missing blocks.
*
* @param imageUUID The image of which you want to get the missing blocks from
* @param amount The amount of blocks that you want to get
* @return The missing blocks
* @throws UploadException If a block was transmitted to many times.
*/
private static List<Integer> getNMissingBlocks( UploadingImage image, int amount ) throws UploadException
{
int missing = image.getAmountOfBlocksNeedingRequest();
log.debug( "The number of missing blocks: " + missing );
if ( amount > missing )
amount = missing;
List<Integer> result = new ArrayList<>( amount );
int got = 0;
for ( int i = 0; i < image.getNumberOfBlocks(); i++ ) {
if ( image.needsRequest( i ) ) {
int times = image.getTimesTransmitted( i );
if ( times > Globals.getSslTransmitTimes() ) {
log.debug( "Block " + i + " is probably broken." );
throw new UploadException( UploadError.BROKEN_BLOCK, "Block " + i + " was transmitted "
+ times + " and is still not valid." );
}
result.add( i );
got++;
}
if ( got == amount )
break;
}
log.debug( "Returned " + got + " missing blocks." );
return result;
}
/**
* Is triggered when an upload of an image is done.
* Removes image from process list, updates db entry and moves file on hard drive.
*
* @param uuid
*/
private static void uploadDone( String uuid )
{
synchronized ( imagesToCheck ) {
imagesToCheck.remove( uuid );
log.debug( "Removing " + uuid );
}
UploadingImage image;
synchronized ( uploadingImages ) {
image = uploadingImages.remove( uuid );
}
image.getDbImage().updateMissingBlocks( null );
// file was already downloaded in the right location by the updownloader class.
// remove the connection so that it can be used by a new client
ConnectionHandler.removeConnection( image.getToken() );
}
public static List<UploadingImage> getImagesToCheck()
{
List<UploadingImage> result = new LinkedList<>();
Iterator<String> iter = imagesToCheck.iterator();
log.debug( imagesToCheck );
while ( iter.hasNext() ) {
result.add( uploadingImages.get( iter.next() ) );
}
return result;
}
public static List<Integer> getRequestedBlocks( String token )
{
// for the downloader
// TODO
return null;
}
/**
* Generates the filePath of an image.
* And creates the folder if wanted.
* The crc file is found under filePath + ".crc"
*
* @param imageData The data of the image
* @param createFolder If you want the folder to be created
* @return The filePath of the given image
*/
public static String generateFilepathOfImage( ImageData imageData )
{
return generateFilePathOfImage( imageData.uuid, imageData.imageName, imageData.imageVersion );
}
public static String generateFilePathOfImage( String uuid, String imageName, int imageVersion) {
String result = Globals.getImageDir() + "/" + uuid + "/";
File dir = new File(result);
if ( !dir.exists() ) {
dir.mkdirs();
}
result += imageName + "-v" + String.valueOf( imageVersion ) + ".vmdk";
return result;
}
public static String generateFilepathOfCrcFile( ImageData imageData )
{
return generateFilepathOfImage( imageData ) + ".crc";
}
public static String generateFilepathOfCrcFile( String uuid, String imageName, int imageVersion )
{
return generateFilePathOfImage( uuid, imageName, imageVersion ) + ".crc";
}
/**
* Checks pending uploads in database and adds them to process list again.
*/
static
{
List<DbImage> list = DbImage.getUploadingImages();
for ( DbImage image : list ) {
UploadingImage infos = new UploadingImage( image.token, image.blockStatus, image.timestamp.getTime(), image.uuid, image.imagePath );
ConnectionHandler.addConnection( image.token, image.imagePath, Connection.UPLOADING ).image = infos;
CrcFile crcFile = new CrcFile( generateFilepathOfCrcFile( image.uuid, image.imageName, image.imageVersion ) ); // TODO: has to be adjusted with the corresponding value above
try {
if ( !crcFile.isValid() ) {
continue;
// UploadingImage object will contain a CRCFile = null which invokes the ImageProcessor to retry to save it
}
} catch ( IOException e ) {
continue;
// same thing here
}
infos.setCrcFile( crcFile );
uploadingImages.put( image.uuid, infos );
imagesToCheck.add( image.uuid );
}
log.info( "Added " + list.size() + " pending upload(s) to process list again." );
}
public static void init()
{
}
}