Moved file chunk helper classes from dozmod-server to shared lib

2 files changed, 190 insertions, 0 deletions

diff --git a/src/main/java/org/openslx/filetransfer/util/ChunkList.java b/src/main/java/org/openslx/filetransfer/util/ChunkList.java
new file mode 100644
index 0000000..9154dc8
--- /dev/null
+++ b/src/main/java/org/openslx/filetransfer/util/ChunkList.java
@@ -0,0 +1,113 @@
+package org.openslx.filetransfer.util;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.log4j.Logger;
+
+public class ChunkList {
+
+	private static final Logger LOGGER = Logger.getLogger(ChunkList.class);
+
+	/**
+	 * Chunks that are missing from the file
+	 */
+	private final List<FileChunk> missingChunks = new LinkedList<>();
+
+	/**
+	 * Chunks that are currently being uploaded or hash-checked
+	 */
+	private final List<FileChunk> pendingChunks = new LinkedList<>();
+
+	private final List<FileChunk> completeChunks = new ArrayList<>(100);
+
+	// 0 = complete, 1 = missing, 2 = uploading, 3 = queued for copying, 4 = copying
+	private final ByteBuffer statusArray;
+
+	// Do we need to keep valid chunks, or chunks that failed too many times?
+
+	public ChunkList(long fileSize, List<ByteBuffer> sha1Sums) {
+		FileChunk.createChunkList(missingChunks, fileSize, sha1Sums);
+		statusArray = ByteBuffer.allocate(missingChunks.size());
+	}
+
+	/**
+	 * Get a missing chunk, marking it pending.
+	 * 
+	 * @return chunk marked as missing
+	 */
+	public synchronized FileChunk getMissing() {
+		if (missingChunks.isEmpty())
+			return null;
+		FileChunk c = missingChunks.remove(0);
+		pendingChunks.add(c);
+		return c;
+	}
+
+	/**
+	 * Get the block status as byte representation.
+	 */
+	public synchronized ByteBuffer getStatusArray() {
+		byte[] array = statusArray.array();
+		//Arrays.fill(array, (byte)0);
+		for (FileChunk c : missingChunks) {
+			array[c.getChunkIndex()] = 1;
+		}
+		for (FileChunk c : pendingChunks) {
+			array[c.getChunkIndex()] = 2;
+		}
+		for (FileChunk c : completeChunks) {
+			array[c.getChunkIndex()] = 0;
+		}
+		return statusArray;
+	}
+
+	/**
+	 * Get completed chunks as list
+	 * 
+	 * @return List containing all successfully transfered chunks
+	 */
+	public synchronized List<FileChunk> getCompleted() {
+		return new ArrayList<>(completeChunks);
+	}
+
+	/**
+	 * Mark a chunk currently transferring as successfully transfered.
+	 * 
+	 * @param c The chunk in question
+	 */
+	public synchronized void markSuccessful(FileChunk c) {
+		if (!pendingChunks.remove(c)) {
+			LOGGER.warn("Inconsistent state: markTransferred called for Chunk " + c.toString()
+					+ ", but chunk is not marked as currently transferring!");
+			return;
+		}
+		completeChunks.add(c);
+	}
+
+	/**
+	 * Mark a chunk currently transferring or being hash checked as failed
+	 * transfer. This increases its fail count and re-adds it to the list of
+	 * missing chunks.
+	 * 
+	 * @param c The chunk in question
+	 * @return Number of times transfer of this chunk failed
+	 */
+	public synchronized int markFailed(FileChunk c) {
+		if (!pendingChunks.remove(c)) {
+			LOGGER.warn("Inconsistent state: markTransferred called for Chunk " + c.toString()
+					+ ", but chunk is not marked as currently transferring!");
+			return -1;
+		}
+		// Add as first element so it will be re-transmitted immediately
+		missingChunks.add(0, c);
+		return c.incFailed();
+	}
+
+	public synchronized boolean isComplete() {
+		return missingChunks.isEmpty() && pendingChunks.isEmpty();
+	}
+
+}
diff --git a/src/main/java/org/openslx/filetransfer/util/FileChunk.java b/src/main/java/org/openslx/filetransfer/util/FileChunk.java
new file mode 100644
index 0000000..3e89b84
--- /dev/null
+++ b/src/main/java/org/openslx/filetransfer/util/FileChunk.java
@@ -0,0 +1,77 @@
+package org.openslx.filetransfer.util;
+
+import java.nio.ByteBuffer;
+import java.util.Collection;
+import java.util.List;
+
+import org.openslx.filetransfer.FileRange;
+
+public class FileChunk {
+
+	public static final int CHUNK_SIZE_MIB = 16;
+	public static final int CHUNK_SIZE = CHUNK_SIZE_MIB * (1024 * 1024);
+
+	public final FileRange range;
+	public final byte[] sha1sum;
+	private int failCount = 0;
+
+	public FileChunk(long startOffset, long endOffset, byte[] sha1sum) {
+		this.range = new FileRange(startOffset, endOffset);
+		this.sha1sum = sha1sum;
+	}
+
+	/**
+	 * Signal that transferring this chunk seems to have failed (checksum
+	 * mismatch).
+	 * 
+	 * @return Number of times the transfer failed now
+	 */
+	public synchronized int incFailed() {
+		return ++failCount;
+	}
+	
+	public int getChunkIndex() {
+		return (int)(range.startOffset / CHUNK_SIZE);
+	}
+	
+	@Override
+	public String toString() {
+		return "[Chunk " + getChunkIndex() + " (" + range.startOffset + "-" + range.endOffset + "), fails: " + failCount + "]";
+	}
+
+	//
+
+	public static int fileSizeToChunkCount(long fileSize) {
+		return (int) ((fileSize + CHUNK_SIZE - 1) / CHUNK_SIZE);
+	}
+
+	public static void createChunkList(Collection<FileChunk> list, long fileSize, List<ByteBuffer> sha1Sums) {
+		if (fileSize < 0)
+			throw new IllegalArgumentException("fileSize cannot be negative");
+		if (!list.isEmpty())
+			throw new IllegalArgumentException("Passed list is not empty");
+		long chunkCount = fileSizeToChunkCount(fileSize);
+		if (sha1Sums != null) {
+			if (sha1Sums.size() != chunkCount)
+				throw new IllegalArgumentException(
+						"Passed a sha1sum list, but hash count in list doesn't match expected chunk count");
+			long offset = 0;
+			for (ByteBuffer sha1sum : sha1Sums) { // Do this as we don't know how efficient List.get(index) is...
+				long end = offset + CHUNK_SIZE;
+				if (end > fileSize)
+					end = fileSize;
+				list.add(new FileChunk(offset, end, sha1sum.array()));
+				offset = end;
+			}
+			return;
+		}
+		long offset = 0;
+		while (offset < fileSize) { // ...otherwise we could share this code
+			long end = offset + CHUNK_SIZE;
+			if (end > fileSize)
+				end = fileSize;
+			list.add(new FileChunk(offset, end, null));
+			offset = end;
+		}
+	}
+}