package org.openslx.imagemaster.serverconnection; import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.HashMap; import java.util.Iterator; import java.util.LinkedList; import java.util.List; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import org.apache.log4j.Logger; import org.openslx.imagemaster.Globals; import org.openslx.imagemaster.crcchecker.CrcFile; import org.openslx.imagemaster.db.DbImage; import org.openslx.imagemaster.db.DbUser; import org.openslx.imagemaster.thrift.iface.DownloadInfos; import org.openslx.imagemaster.thrift.iface.ImageData; import org.openslx.imagemaster.thrift.iface.ImageDataError; import org.openslx.imagemaster.thrift.iface.ImageDataException; import org.openslx.imagemaster.thrift.iface.UploadError; import org.openslx.imagemaster.thrift.iface.UploadException; import org.openslx.imagemaster.thrift.iface.UploadInfos; import org.openslx.imagemaster.util.RandomString; import org.openslx.imagemaster.util.Util; /** * Processing the up- and download of images. * Handles who is authorized and knows which blocks are missing / need to be sent */ public class ImageProcessor { private static final Logger log = Logger.getLogger( ImageProcessor.class ); /** * The amount of blocks that is return in UploadInfos (after request of satellite) */ private static final int AMOUNT = 20; /** * The uploading images. * Key: imageUUID, * Value: uploadingImageInfos */ private static Map uploadingImages = new ConcurrentHashMap<>(); /** * The UUIDs of the images that need to be checked by the crc checker. */ private static List imagesToCheck = new LinkedList<>(); /** * The downloading clients. * Key: serverSessionId * Value: downloadingClientInfos */ private static HashMap downloadingClients = new HashMap<>(); /** * Checks if this image is already uploading and returns a new list with missing blocks if so. * Puts the new image into processing list else. * * @param serverSessionId The uploading server * @param imageData The data of the image * @return * @throws UploadException If some error occurred during the process */ public static UploadInfos getUploadInfos( String serverSessionId, ImageData imageData, List crcSums ) throws UploadException, ImageDataException { // check image data if ( imageData.imageName == null || imageData.imageName.isEmpty() ) { throw new ImageDataException( ImageDataError.INVALID_DATA, "Image name not set." ); } else if ( imageData.imageOwner == null || imageData.imageOwner.isEmpty() ) { throw new ImageDataException( ImageDataError.INVALID_DATA, "Image owner not set." ); } else if ( imageData.contentOperatingSystem == null || imageData.contentOperatingSystem.isEmpty() ) { throw new ImageDataException( ImageDataError.INVALID_DATA, "Content operating system not set." ); } else if ( imageData.fileSize <= 0 ) { throw new ImageDataException( ImageDataError.INVALID_DATA, "File size is too small." ); } else if ( !DbUser.exists( imageData.imageOwner ) ) { throw new ImageDataException( ImageDataError.INVALID_DATA, "User is not known." ); } //TODO: this is not working like this: DbImage i = DbImage.getImageByUuid( imageData.uuid ); // boolean isUpdate = false; // if ( i != null ) { // // image is already available // // is the client updating?? // if ( imageData.imageVersion <= i.imageVersion ) { // throw new ImageDataException( ImageDataError.INVALID_DATA, "This image with the same or a newer version is already available." ); // } else { // // TODO: update db and prepare for new image file // isUpdate = true; // } // } log.debug( serverSessionId + " is submitting " + imageData.uuid ); String uuid = imageData.uuid; String token; String filepath; int nBlocks; int[] allBlocks; UploadingImage image; synchronized ( uploadingImages ) { // check if image is already uploading if ( ( image = uploadingImages.get( uuid ) ) != null ) { if ( image.getCrcFile() == null ) { CrcFile crcFile; try { // try to write crc file ... crcFile = CrcFile.writeCrcFile( crcSums, generateFilepathOfCrcFile( imageData ) ); } catch ( IOException e ) { // ... and keep it in ram if it fails crcFile = new CrcFile( crcSums ); } image.setCrcFile( crcFile ); } List missing = getNMissingBlocks( image, AMOUNT ); if ( missing.isEmpty() && image.allBlocksValid() ) { uploadDone( uuid ); return new UploadInfos( image.getToken(), Globals.getSslSocketPort(), missing, image.allBlocksValid() ); } return new UploadInfos( image.getToken(), Globals.getSslSocketPort(), missing, false ); } // insert new image if ( !CrcFile.sumsAreValid( crcSums ) ) throw new UploadException( UploadError.INVALID_CRC, "CRC sums were invalid." ); filepath = generateFilepathOfImage( imageData ); token = RandomString.generate( 100, false ); nBlocks = Util.getNumberOfBlocks( imageData.fileSize, Globals.blockSize ); allBlocks = new int[ nBlocks ]; // initialize array with all zeros which mean that this block is missing image = new UploadingImage( token, allBlocks, System.currentTimeMillis(), uuid, filepath ); image.setDbImage( i ); // set the dbImage (it doesn't matter if the image is null because the uploadingImage is creating it then uploadingImages.put( uuid, image ); } CrcFile crcFile; try { // try to write crc file ... crcFile = CrcFile.writeCrcFile( crcSums, generateFilepathOfCrcFile( imageData ) ); } catch ( IOException e ) { // ... and keep it in ram if it fails crcFile = new CrcFile( crcSums ); } image.setCrcFile( crcFile ); ConnectionHandler.addConnection( token, filepath, Connection.UPLOADING ).image = image; // if ( isUpdate ) { // i.updateVersion( i.imageVersion, Util.getNumberOfBlocks( i.fileSize, Globals.blockSize ) ); // } else { DbImage.insert( imageData, System.currentTimeMillis(), token, nBlocks, serverSessionId, filepath ); // } imagesToCheck.add( uuid ); log.debug( imagesToCheck ); log.debug( image.toString() ); return new UploadInfos( token, Globals.getSslSocketPort(), getNMissingBlocks( image, AMOUNT ), false ); } public static DownloadInfos getDownloadInfos( String serverSessionId, String uuid, List requestedBlocks ) { // check if server is already downloading if ( downloadingClients.containsKey( serverSessionId ) ) { DownloadingClient client = downloadingClients.get( serverSessionId ); // remove download if done if ( requestedBlocks.isEmpty() ) { downloadDone( serverSessionId, uuid ); return new DownloadInfos(); } if ( client.isDownloading( uuid ) ) { // client was downloading this image // update the requested blocks client.requestBlocks( uuid, requestedBlocks ); return new DownloadInfos( client.getToken( uuid ), Globals.getSslSocketPort() ); } // server was downloading another image and now gets a new connection for this new download String token = RandomString.generate( 100, false ); String filepath = DbImage.getImageByUuid( uuid ).imagePath; ConnectionHandler.addConnection( token, filepath, Connection.DOWNLOADING ); client.addDownload( uuid, requestedBlocks, token ); downloadingClients.put( serverSessionId, client ); return new DownloadInfos( token, Globals.getSslSocketPort() ); } // insert new client and start listener synchronized ( downloadingClients ) { String token = RandomString.generate( 100, false ); String filepath = DbImage.getImageByUuid( uuid ).imagePath; DownloadingClient client = new DownloadingClient(); client.addDownload( uuid, requestedBlocks, token ); downloadingClients.put( serverSessionId, client ); ConnectionHandler.addConnection( token, filepath, Connection.DOWNLOADING ).client = client; return new DownloadInfos( token, Globals.getSslSocketPort() ); } } private static void downloadDone( String serverSessionId, String uuid ) { synchronized ( downloadingClients ) { DownloadingClient client = downloadingClients.get( serverSessionId ); client.removeDownload( uuid ); ConnectionHandler.removeConnection( client.getToken( uuid ) ); if ( !client.hasDownloads() ) { downloadingClients.remove( serverSessionId ); } } } /** * Returns a specified number of missing blocks. * * @param imageUUID The image of which you want to get the missing blocks from * @param amount The amount of blocks that you want to get * @return The missing blocks * @throws UploadException If a block was transmitted to many times. */ private static List getNMissingBlocks( UploadingImage image, int amount ) throws UploadException { int missing = image.getAmountOfBlocksNeedingRequest(); log.debug( "The number of missing blocks: " + missing ); if ( amount > missing ) amount = missing; List result = new ArrayList<>( amount ); int got = 0; for ( int i = 0; i < image.getNumberOfBlocks(); i++ ) { if ( image.needsRequest( i ) ) { int times = image.getTimesTransmitted( i ); if ( times > Globals.getSslTransmitTimes() ) { log.debug( "Block " + i + " is probably broken." ); throw new UploadException( UploadError.BROKEN_BLOCK, "Block " + i + " was transmitted " + times + " and is still not valid." ); } result.add( i ); got++; } if ( got == amount ) break; } log.debug( "Returned " + got + " missing blocks." ); return result; } /** * Is triggered when an upload of an image is done. * Removes image from process list, updates db entry and moves file on hard drive. * * @param uuid */ private static void uploadDone( String uuid ) { synchronized ( imagesToCheck ) { imagesToCheck.remove( uuid ); log.debug( "Removing " + uuid ); } UploadingImage image; synchronized ( uploadingImages ) { image = uploadingImages.remove( uuid ); } image.getDbImage().updateMissingBlocks( null ); // file was already downloaded in the right location by the updownloader class. // remove the connection so that it can be used by a new client ConnectionHandler.removeConnection( image.getToken() ); } public static List getImagesToCheck() { List result = new LinkedList<>(); Iterator iter = imagesToCheck.iterator(); log.debug( imagesToCheck ); while ( iter.hasNext() ) { result.add( uploadingImages.get( iter.next() ) ); } return result; } public static List getRequestedBlocks( String token ) { // for the downloader // TODO return null; } /** * Generates the filePath of an image. * And creates the folder if wanted. * The crc file is found under filePath + ".crc" * * @param imageData The data of the image * @param createFolder If you want the folder to be created * @return The filePath of the given image */ public static String generateFilepathOfImage( ImageData imageData ) { return generateFilePathOfImage( imageData.uuid, imageData.imageName, imageData.imageVersion ); } public static String generateFilePathOfImage( String uuid, String imageName, int imageVersion) { String result = Globals.getImageDir() + "/" + uuid + "/"; File dir = new File(result); if ( !dir.exists() ) { dir.mkdirs(); } result += imageName + "-v" + String.valueOf( imageVersion ) + ".vmdk"; return result; } public static String generateFilepathOfCrcFile( ImageData imageData ) { return generateFilepathOfImage( imageData ) + ".crc"; } public static String generateFilepathOfCrcFile( String uuid, String imageName, int imageVersion ) { return generateFilePathOfImage( uuid, imageName, imageVersion ) + ".crc"; } /** * Checks pending uploads in database and adds them to process list again. */ static { List list = DbImage.getUploadingImages(); for ( DbImage image : list ) { UploadingImage infos = new UploadingImage( image.token, image.blockStatus, image.timestamp.getTime(), image.uuid, image.imagePath ); ConnectionHandler.addConnection( image.token, image.imagePath, Connection.UPLOADING ).image = infos; CrcFile crcFile = new CrcFile( generateFilepathOfCrcFile( image.uuid, image.imageName, image.imageVersion ) ); // TODO: has to be adjusted with the corresponding value above try { if ( !crcFile.isValid() ) { continue; // UploadingImage object will contain a CRCFile = null which invokes the ImageProcessor to retry to save it } } catch ( IOException e ) { continue; // same thing here } infos.setCrcFile( crcFile ); uploadingImages.put( image.uuid, infos ); imagesToCheck.add( image.uuid ); } log.info( "Added " + list.size() + " pending upload(s) to process list again." ); } public static void init() { } }