diff options
author | Simon Rettberg | 2015-07-21 14:24:55 +0200 |
---|---|---|
committer | Simon Rettberg | 2015-07-21 14:24:55 +0200 |
commit | 5e5dc310f69a174e2f8eb3cd579566cb9145af5f (patch) | |
tree | 12869dc9372284dca903c1d6d870a4c050cb8bfb /src/main/java/org/openslx/filetransfer | |
parent | Extended Util class (diff) | |
download | master-sync-shared-5e5dc310f69a174e2f8eb3cd579566cb9145af5f.tar.gz master-sync-shared-5e5dc310f69a174e2f8eb3cd579566cb9145af5f.tar.xz master-sync-shared-5e5dc310f69a174e2f8eb3cd579566cb9145af5f.zip |
Moved file chunk helper classes from dozmod-server to shared lib
Diffstat (limited to 'src/main/java/org/openslx/filetransfer')
-rw-r--r-- | src/main/java/org/openslx/filetransfer/util/ChunkList.java | 113 | ||||
-rw-r--r-- | src/main/java/org/openslx/filetransfer/util/FileChunk.java | 77 |
2 files changed, 190 insertions, 0 deletions
diff --git a/src/main/java/org/openslx/filetransfer/util/ChunkList.java b/src/main/java/org/openslx/filetransfer/util/ChunkList.java new file mode 100644 index 0000000..9154dc8 --- /dev/null +++ b/src/main/java/org/openslx/filetransfer/util/ChunkList.java @@ -0,0 +1,113 @@ +package org.openslx.filetransfer.util; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.LinkedList; +import java.util.List; + +import org.apache.log4j.Logger; + +public class ChunkList { + + private static final Logger LOGGER = Logger.getLogger(ChunkList.class); + + /** + * Chunks that are missing from the file + */ + private final List<FileChunk> missingChunks = new LinkedList<>(); + + /** + * Chunks that are currently being uploaded or hash-checked + */ + private final List<FileChunk> pendingChunks = new LinkedList<>(); + + private final List<FileChunk> completeChunks = new ArrayList<>(100); + + // 0 = complete, 1 = missing, 2 = uploading, 3 = queued for copying, 4 = copying + private final ByteBuffer statusArray; + + // Do we need to keep valid chunks, or chunks that failed too many times? + + public ChunkList(long fileSize, List<ByteBuffer> sha1Sums) { + FileChunk.createChunkList(missingChunks, fileSize, sha1Sums); + statusArray = ByteBuffer.allocate(missingChunks.size()); + } + + /** + * Get a missing chunk, marking it pending. + * + * @return chunk marked as missing + */ + public synchronized FileChunk getMissing() { + if (missingChunks.isEmpty()) + return null; + FileChunk c = missingChunks.remove(0); + pendingChunks.add(c); + return c; + } + + /** + * Get the block status as byte representation. + */ + public synchronized ByteBuffer getStatusArray() { + byte[] array = statusArray.array(); + //Arrays.fill(array, (byte)0); + for (FileChunk c : missingChunks) { + array[c.getChunkIndex()] = 1; + } + for (FileChunk c : pendingChunks) { + array[c.getChunkIndex()] = 2; + } + for (FileChunk c : completeChunks) { + array[c.getChunkIndex()] = 0; + } + return statusArray; + } + + /** + * Get completed chunks as list + * + * @return List containing all successfully transfered chunks + */ + public synchronized List<FileChunk> getCompleted() { + return new ArrayList<>(completeChunks); + } + + /** + * Mark a chunk currently transferring as successfully transfered. + * + * @param c The chunk in question + */ + public synchronized void markSuccessful(FileChunk c) { + if (!pendingChunks.remove(c)) { + LOGGER.warn("Inconsistent state: markTransferred called for Chunk " + c.toString() + + ", but chunk is not marked as currently transferring!"); + return; + } + completeChunks.add(c); + } + + /** + * Mark a chunk currently transferring or being hash checked as failed + * transfer. This increases its fail count and re-adds it to the list of + * missing chunks. + * + * @param c The chunk in question + * @return Number of times transfer of this chunk failed + */ + public synchronized int markFailed(FileChunk c) { + if (!pendingChunks.remove(c)) { + LOGGER.warn("Inconsistent state: markTransferred called for Chunk " + c.toString() + + ", but chunk is not marked as currently transferring!"); + return -1; + } + // Add as first element so it will be re-transmitted immediately + missingChunks.add(0, c); + return c.incFailed(); + } + + public synchronized boolean isComplete() { + return missingChunks.isEmpty() && pendingChunks.isEmpty(); + } + +} diff --git a/src/main/java/org/openslx/filetransfer/util/FileChunk.java b/src/main/java/org/openslx/filetransfer/util/FileChunk.java new file mode 100644 index 0000000..3e89b84 --- /dev/null +++ b/src/main/java/org/openslx/filetransfer/util/FileChunk.java @@ -0,0 +1,77 @@ +package org.openslx.filetransfer.util; + +import java.nio.ByteBuffer; +import java.util.Collection; +import java.util.List; + +import org.openslx.filetransfer.FileRange; + +public class FileChunk { + + public static final int CHUNK_SIZE_MIB = 16; + public static final int CHUNK_SIZE = CHUNK_SIZE_MIB * (1024 * 1024); + + public final FileRange range; + public final byte[] sha1sum; + private int failCount = 0; + + public FileChunk(long startOffset, long endOffset, byte[] sha1sum) { + this.range = new FileRange(startOffset, endOffset); + this.sha1sum = sha1sum; + } + + /** + * Signal that transferring this chunk seems to have failed (checksum + * mismatch). + * + * @return Number of times the transfer failed now + */ + public synchronized int incFailed() { + return ++failCount; + } + + public int getChunkIndex() { + return (int)(range.startOffset / CHUNK_SIZE); + } + + @Override + public String toString() { + return "[Chunk " + getChunkIndex() + " (" + range.startOffset + "-" + range.endOffset + "), fails: " + failCount + "]"; + } + + // + + public static int fileSizeToChunkCount(long fileSize) { + return (int) ((fileSize + CHUNK_SIZE - 1) / CHUNK_SIZE); + } + + public static void createChunkList(Collection<FileChunk> list, long fileSize, List<ByteBuffer> sha1Sums) { + if (fileSize < 0) + throw new IllegalArgumentException("fileSize cannot be negative"); + if (!list.isEmpty()) + throw new IllegalArgumentException("Passed list is not empty"); + long chunkCount = fileSizeToChunkCount(fileSize); + if (sha1Sums != null) { + if (sha1Sums.size() != chunkCount) + throw new IllegalArgumentException( + "Passed a sha1sum list, but hash count in list doesn't match expected chunk count"); + long offset = 0; + for (ByteBuffer sha1sum : sha1Sums) { // Do this as we don't know how efficient List.get(index) is... + long end = offset + CHUNK_SIZE; + if (end > fileSize) + end = fileSize; + list.add(new FileChunk(offset, end, sha1sum.array())); + offset = end; + } + return; + } + long offset = 0; + while (offset < fileSize) { // ...otherwise we could share this code + long end = offset + CHUNK_SIZE; + if (end > fileSize) + end = fileSize; + list.add(new FileChunk(offset, end, null)); + offset = end; + } + } +} |