From 2037cb1a089787adeb3b3c7d83d5d0f1e1502aa0 Mon Sep 17 00:00:00 2001 From: Nils Schwabe Date: Fri, 4 Jul 2014 13:02:57 +0200 Subject: Move crc checler from masterserver to master-sync-shared --- .../openslx/imagemaster/crcchecker/CRCChecker.java | 76 ++++++++++++++++++++++ .../openslx/imagemaster/crcchecker/CRCFile.java | 72 ++++++++++++++++++++ .../openslx/imagemaster/crcchecker/ImageFile.java | 57 ++++++++++++++++ 3 files changed, 205 insertions(+) create mode 100644 src/main/java/org/openslx/imagemaster/crcchecker/CRCChecker.java create mode 100644 src/main/java/org/openslx/imagemaster/crcchecker/CRCFile.java create mode 100644 src/main/java/org/openslx/imagemaster/crcchecker/ImageFile.java (limited to 'src/main/java/org/openslx/imagemaster') diff --git a/src/main/java/org/openslx/imagemaster/crcchecker/CRCChecker.java b/src/main/java/org/openslx/imagemaster/crcchecker/CRCChecker.java new file mode 100644 index 0000000..9ca3d55 --- /dev/null +++ b/src/main/java/org/openslx/imagemaster/crcchecker/CRCChecker.java @@ -0,0 +1,76 @@ +package org.openslx.imagemaster.crcchecker; + +import java.io.IOException; +import java.util.LinkedList; +import java.util.List; +import java.util.zip.CRC32; + +import org.apache.log4j.Logger; + +// TODO: Move to master-sync-shared, sattelite daemon might want to check images too + +public class CRCChecker +{ + + private static Logger log = Logger.getLogger( CRCChecker.class ); + private static final int blockSize = 16 * 1024 * 1024; + + /** + * Checks the CRC sum of given blocks of a given imageFile against a given crcFile. + * The caller needs to make sure that block that are going to be checked are complete! + * + * @param imageFile The imageFile to check + * @param crcFile The crcFile to check against + * @param blocks The blocks to check + * @return List of blocks where the crc matches, or null if the crc file is corrupted + */ + public static List checkCRC( String imageFile, String crcFile, List blocks ) throws IOException + { + List result = new LinkedList<>(); + + ImageFile image = new ImageFile( imageFile, blockSize ); + CRCFile crc = new CRCFile( crcFile ); + + log.debug( "Checking image file: '" + imageFile + "' with crc file: '" + crcFile + "'" ); + try { + if ( !crc.isValid() ) + return null; + // TODO: also return null if the crc file contains the wrong number of checksums (only makes sense if the upload is complete) + } catch ( IOException e ) { + throw new IOException( "Could not read CRC file", e ); + } + + // check all blocks + byte[] block = new byte[blockSize]; + for ( Integer blockN : blocks ) { + try { + image.getBlock( blockN, block ); + } catch ( IOException e ) { + throw new IOException( "Could not read image file", e ); + } + + if ( block == null ) + continue; // some error occured (for example: someone tried to check a block that is not in the file) + + // check this block with CRC32 + // add this block to result, if check was ok with CRC file + + CRC32 crcCalc = new CRC32(); + crcCalc.update( block ); + int crcSum = Integer.reverseBytes( (int)crcCalc.getValue() ); + int crcSumFromFile; + try { + crcSumFromFile = crc.getCRCSum( blockN ); + } catch ( IOException e ) { + throw new IOException( "Could not read CRC file", e ); + } + + if ( crcSum == crcSumFromFile ) + result.add( blockN ); + else + log.debug( blockN + " was invalid" ); + } + + return result; + } +} diff --git a/src/main/java/org/openslx/imagemaster/crcchecker/CRCFile.java b/src/main/java/org/openslx/imagemaster/crcchecker/CRCFile.java new file mode 100644 index 0000000..b557ada --- /dev/null +++ b/src/main/java/org/openslx/imagemaster/crcchecker/CRCFile.java @@ -0,0 +1,72 @@ +package org.openslx.imagemaster.crcchecker; + +import java.io.DataInputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.zip.CRC32; + +/** + * Represents a crcfile + * @author nils + * + */ +public class CRCFile +{ + private File file; + private List crcSums = null; + + CRCFile(String filename) { + this.file = new File( filename ); + } + + /** + * Checks if this crc file is valid. + * (If the crc over the file is equal to the first crc sum.) + * @return Whether the crc file is valid + * @throws IOException If the file could not be read or could not be found + */ + public boolean isValid() throws IOException { + FileInputStream fis = new FileInputStream( file ); + DataInputStream dis = new DataInputStream( fis ); + + int crcSum = dis.readInt(); + + CRC32 crcCalc = new CRC32(); + + byte[] bytes = new byte[(int) file.length() - Integer.SIZE/8]; // byte array with length of the file minus the first crc sum (=4byte) + fis.read( bytes ); + crcCalc.update( bytes ); + + dis.close(); + + if (crcSum == Integer.reverseBytes( (int) crcCalc.getValue() ) ) return true; + else return false; + } + + /** + * Get a specified crcSum for a block number + * @param blockNumber + * @return The crcSum or 0 if the block number is invalid + * @throws IOException + */ + public int getCRCSum(int blockNumber) throws IOException { + if (crcSums == null) { + // the crcSums were not read yet + DataInputStream dis = new DataInputStream( new FileInputStream( file ) ); + crcSums = new ArrayList<>(); + for (int i = 0; i < file.length()/4; i++) { + int s = dis.readInt(); + if (i > 0) crcSums.add( s ); // skip the first crcSum because it's the sum over the crcFile + } + dis.close(); + } + + if (blockNumber < 0) return 0; + if (blockNumber > crcSums.size() - 1) return 0; + + return crcSums.get( blockNumber ); + } +} diff --git a/src/main/java/org/openslx/imagemaster/crcchecker/ImageFile.java b/src/main/java/org/openslx/imagemaster/crcchecker/ImageFile.java new file mode 100644 index 0000000..cb8c700 --- /dev/null +++ b/src/main/java/org/openslx/imagemaster/crcchecker/ImageFile.java @@ -0,0 +1,57 @@ +package org.openslx.imagemaster.crcchecker; + +import java.io.File; +import java.io.IOException; +import java.io.RandomAccessFile; + +/** + * Representing an image file. + * Is able to return certain blocks of this file. + * @author nils + * + */ +public class ImageFile +{ + private File f; + private RandomAccessFile file = null; + private int blockSize; + + public ImageFile(String filename, int blockSize) { + this.f = new File( filename ); + this.blockSize = blockSize; + } + + /** + * Get a certain block (uses RandomAccessFile) + * If the last block is not full an array with a smaller size is set + * and the actual number of bytes is returned. + * @param block The number of the block you want to get + * @return The actual size of the array or 0 if the block number is < 0 or the block is not in the file + * @throws IOException When file was not found or could not be read + */ + public int getBlock(int block, byte[] array) throws IOException { + if (block < 0) return 0; + if (block > f.length()/blockSize) return 0; + + if (file == null) { + file = new RandomAccessFile( f, "r" ); + } + + file.seek( (long)block * blockSize ); + long remaining = length() - (block * blockSize); + + if (blockSize > remaining) { + array = new byte[(int)remaining]; // only read available bytes + file.read( array ); + return (int)remaining; // return actual size of array + } else { + // size of array is ok, read the full array and return block size + file.read( array ); + return this.blockSize; + } + } + + public long length() { + return f.length(); + } +} \ No newline at end of file -- cgit v1.2.3-55-g7522