[SERVER] Copy CRC-32 list from uplink server if available

Split up helper.c, move file/disk related functions to fileutil.c
Uplink: Make sure relayed requests are at least 1MiB

15 files changed, 299 insertions, 117 deletions

diff --git a/src/config.h b/src/config.h
index 7a22c01..76b131b 100644
--- a/src/config.h
+++ b/src/config.h
@@ -37,7 +37,7 @@
 #define SERVER_RTT_DELAY_MAX 15
 
 #define SERVER_REMOTE_IMAGE_CHECK_CACHETIME 600 // 10 minutes
-
+#define SERVER_MAX_PROXY_IMAGE_SIZE 100000000000LL // 100GB
 // +++++ Network +++++
 // Default port
 #define PORT 5003
diff --git a/src/server/altservers.c b/src/server/altservers.c
index 84eb0db..1410615 100644
--- a/src/server/altservers.c
+++ b/src/server/altservers.c
@@ -57,7 +57,7 @@ int altservers_load()
 {
 	int count = 0;
 	char *name = NULL, *space;
-	char line[1000];
+	char buffer[1000], *line;
 	dnbd3_host_t host;
 	asprintf( &name, "%s/%s", _configDir, "alt-servers" );
 	if ( name == NULL ) return -1;
@@ -65,7 +65,13 @@ int altservers_load()
 	free( name );
 	if ( fp == NULL ) return -1;
 	while ( !feof( fp ) ) {
-		if ( fgets( line, 1000, fp ) == NULL ) break;
+		if ( fgets( buffer, 1000, fp ) == NULL ) break;
+		int isPrivate = FALSE;
+		for (line = buffer;;) { // Trim left and scan for "-" prefix
+			if ( *line == '-' ) isPrivate = TRUE;
+			else if ( *line != ' ' || *line != '\t' ) break;
+			line++;
+		}
 		trim_right( line );
 		space = strchr( line, ' ' );
 		if ( space != NULL ) *space++ = '\0';
@@ -74,14 +80,14 @@ int altservers_load()
 			memlogf( "[WARNING] Invalid entry in alt-servers file ignored: '%s'", line );
 			continue;
 		}
-		if ( altservers_add( &host, space ) ) ++count;
+		if ( altservers_add( &host, space, isPrivate ) ) ++count;
 	}
 	fclose( fp );
 	printf( "[DEBUG] Added %d alt servers\n", count );
 	return count;
 }
 
-int altservers_add(dnbd3_host_t *host, const char *comment)
+int altservers_add(dnbd3_host_t *host, const char *comment, const int isPrivate)
 {
 	int i, freeSlot = -1;
 	spin_lock( &_alts_lock );
@@ -102,6 +108,7 @@ int altservers_add(dnbd3_host_t *host, const char *comment)
 		freeSlot = _num_alts++;
 	}
 	_alt_servers[freeSlot].host = *host;
+	_alt_servers[freeSlot].isPrivate = isPrivate;
 	if ( comment != NULL ) snprintf( _alt_servers[freeSlot].comment, COMMENT_LENGTH, "%s", comment );
 	spin_unlock( &_alts_lock );
 	return TRUE;
@@ -150,6 +157,7 @@ void altservers_removeUplink(dnbd3_connection_t *uplink)
 /**
  * Get <size> known (working) alt servers, ordered by network closeness
  * (by finding the smallest possible subnet)
+ * Private servers are excluded
  */
 int altservers_getMatching(dnbd3_host_t *host, dnbd3_server_entry_t *output, int size)
 {
@@ -160,6 +168,7 @@ int altservers_getMatching(dnbd3_host_t *host, dnbd3_server_entry_t *output, int
 	spin_lock( &_alts_lock );
 	for (i = 0; i < _num_alts; ++i) {
 		if ( host->type != _alt_servers[i].host.type ) continue; // Wrong address family
+		if ( _alt_servers[i].isPrivate ) continue; // Do not tell clients about private servers
 		// TODO: Prefer same AF here, but if in the end we got less servers than requested, add
 		// servers of other AF too (after this loop)
 		if ( count == 0 ) {
@@ -324,7 +333,7 @@ static void *altservers_main(void *data)
 		// Empty pipe
 		do {
 			ret = read( readPipe, buffer, sizeof buffer );
-		} while ( ret > 0 ); // Throw data away, this is just used for waking this thread up
+		} while ( ret == sizeof buffer ); // Throw data away, this is just used for waking this thread up
 		if ( ret == 0 ) {
 			memlogf( "[WARNING] Signal pipe of uplink_connector for %s closed! Things will break!" );
 		}
@@ -390,22 +399,25 @@ static void *altservers_main(void *data)
 				if ( !dnbd3_get_block( sock,
 				        (((uint64_t)start.tv_nsec | (uint64_t)rand()) * DNBD3_BLOCK_SIZE )% uplink->image->filesize,
 				        DNBD3_BLOCK_SIZE) ) {
-					ERROR_GOTO_VA( server_failed, "[ERROR] Could not request random block for %s", uplink->image->lower_name );
+					//ERROR_GOTO_VA( server_failed, "[ERROR] Could not request random block for %s", uplink->image->lower_name );
+					goto server_failed;
 				}
 				// See if requesting the block succeeded ++++++++++++++++++++++
 				if ( !dnbd3_get_reply( sock, &reply ) ) {
 					char buf[100] = { 0 };
 					host_to_string( &servers[itAlt], buf, 100 );
-					ERROR_GOTO_VA( server_failed, "[ERROR] Received corrupted reply header (%s) after CMD_GET_BLOCK (%s)",
-					        buf, uplink->image->lower_name );
+					//ERROR_GOTO_VA( server_failed, "[ERROR] Received corrupted reply header (%s) after CMD_GET_BLOCK (%s)",
+					//        buf, uplink->image->lower_name );
+					goto server_failed;
 				}
 				// check reply header
 				if ( reply.cmd != CMD_GET_BLOCK || reply.size != DNBD3_BLOCK_SIZE ) {
-					ERROR_GOTO_VA( server_failed, "[ERROR] Reply to random block request is %d bytes for %s",
-					        reply.size, uplink->image->lower_name );
+					//ERROR_GOTO_VA( server_failed, "[ERROR] Reply to random block request is %d bytes for %s",
+					//        reply.size, uplink->image->lower_name );
+					goto server_failed;
 				}
 				if ( recv( sock, buffer, DNBD3_BLOCK_SIZE, MSG_WAITALL ) != DNBD3_BLOCK_SIZE ) {
-					ERROR_GOTO_VA( server_failed, "[ERROR] Could not read random block from socket for %s", uplink->image->lower_name );
+					ERROR_GOTO_VA( server_failed, "[ERROR] Could not read random block payload for %s", uplink->image->lower_name );
 				}
 				clock_gettime( CLOCK_MONOTONIC_RAW, &end );
 				// Measurement done - everything fine so far
diff --git a/src/server/altservers.h b/src/server/altservers.h
index bb09b73..e826946 100644
--- a/src/server/altservers.h
+++ b/src/server/altservers.h
@@ -7,7 +7,7 @@ void altservers_init();
 
 int altservers_load();
 
-int altservers_add(dnbd3_host_t *host, const char *comment);
+int altservers_add(dnbd3_host_t *host, const char *comment, const int isPrivate);
 
 void altservers_findUplink(dnbd3_connection_t *uplink);
 
diff --git a/src/server/fileutil.c b/src/server/fileutil.c
new file mode 100644
index 0000000..ea247f0
--- /dev/null
+++ b/src/server/fileutil.c
@@ -0,0 +1,75 @@
+#include "fileutil.h"
+
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <sys/statvfs.h>
+#include <errno.h>
+#include <stdlib.h>
+#include <assert.h>
+#include <string.h>
+
+int file_isReadable(char *file)
+{
+	int fd = open( file, O_RDONLY );
+	if ( fd < 0 ) return FALSE;
+	close( fd );
+	return TRUE;
+}
+
+int file_isWritable(char *file)
+{
+	int fd = open( file, O_WRONLY );
+	if ( fd >= 0 ) {
+		close( fd );
+		return TRUE;
+	}
+	fd = open( file, O_WRONLY | O_CREAT, 0600 );
+	if ( fd < 0 ) return FALSE;
+	close( fd );
+	unlink( file );
+	return TRUE;
+}
+
+int mkdir_p(const char* path)
+{
+	assert( path != NULL );
+	if ( *path == '\0' ) return TRUE;
+	char buffer[strlen( path ) + 1];
+	strcpy( buffer, path );
+	char *current = buffer;
+	char *slash;
+	while ( (slash = strchr( current, '/' )) != NULL ) {
+		*slash = '\0';
+		if ( *buffer != '\0' && mkdir( buffer, 0750 ) != 0 && errno != EEXIST ) return FALSE;
+		*slash = '/';
+		current = slash + 1;
+	}
+	if ( mkdir( buffer, 0750 ) != 0 && errno != EEXIST ) return FALSE;
+	return TRUE;
+}
+
+int file_alloc(int fd, uint64_t offset, uint64_t size)
+{
+	if ( fallocate( fd, 0, offset, size ) == 0 ) return TRUE; // fast way
+	if ( posix_fallocate( fd, offset, size ) == 0 ) return TRUE; // slow way
+	if ( lseek( fd, offset + size - 1, SEEK_SET ) != offset ) return FALSE; // dumb way
+	if ( write( fd, "", 1 ) != 1 ) return FALSE;
+	return TRUE;
+}
+
+uint64_t file_freeDiskSpace(const char * const path)
+{
+	struct statvfs fiData;
+	if ( (statvfs( path, &fiData )) < 0 ) {
+		return 0;
+	}
+	return ((uint64_t)fiData.f_bfree * (uint64_t)fiData.f_bsize * 95LL) / 100LL; // Assume 5% root reservation is active
+}
+
+time_t file_lastModification(const char * const file)
+{
+	struct stat st;
+	if ( stat( file, &st ) != 0 ) return 0;
+	return st.st_mtime;
+}
diff --git a/src/server/fileutil.h b/src/server/fileutil.h
new file mode 100644
index 0000000..394338b
--- /dev/null
+++ b/src/server/fileutil.h
@@ -0,0 +1,14 @@
+#ifndef _FILEUTIL_H_
+#define _FILEUTIL_H_
+
+#include "../types.h"
+#include <time.h>
+
+int file_isReadable(char *file);
+int file_isWritable(char *file);
+int mkdir_p(const char* path);
+int file_alloc(int fd, uint64_t offset, uint64_t size);
+uint64_t file_freeDiskSpace(const char * const path);
+time_t file_lastModification(const char * const file);
+
+#endif /* FILEUTIL_H_ */
diff --git a/src/server/globals.c b/src/server/globals.c
index 1e23fb3..ac3f279 100644
--- a/src/server/globals.c
+++ b/src/server/globals.c
@@ -12,6 +12,7 @@ int _vmdkLegacyMode = FALSE;
 int _shutdown = 0;
 int _serverPenalty = 0;
 int _clientPenalty = 0;
+int _isProxy = FALSE;
 
 #define SAVE_TO_VAR_STR(ss, kk) do { if (strcmp(section, #ss) == 0 && strcmp(key, #kk) == 0) { if (_ ## kk != NULL) free(_ ## kk); _ ## kk = strdup(value); } } while (0)
 #define SAVE_TO_VAR_BOOL(ss, kk) do { if (strcmp(section, #ss) == 0 && strcmp(key, #kk) == 0) _ ## kk = atoi(value) != 0 || strcmp(value, "true") == 0 || strcmp(value, "True") == 0 || strcmp(value, "TRUE") == 0; } while (0)
@@ -19,8 +20,9 @@ int _clientPenalty = 0;
 
 static int ini_handler(void *custom, const char* section, const char* key, const char* value)
 {
-	SAVE_TO_VAR_STR( dnbd3, basePath );
+	if ( _basePath == NULL ) SAVE_TO_VAR_STR( dnbd3, basePath );
 	SAVE_TO_VAR_BOOL( dnbd3, vmdkLegacyMode );
+	SAVE_TO_VAR_BOOL( dnbd3, isProxy );
 	SAVE_TO_VAR_INT( dnbd3, serverPenalty );
 	SAVE_TO_VAR_INT( dnbd3, clientPenalty );
 	return TRUE;
diff --git a/src/server/globals.h b/src/server/globals.h
index 2159021..f53cb11 100644
--- a/src/server/globals.h
+++ b/src/server/globals.h
@@ -62,7 +62,7 @@ typedef struct
 	uint8_t data[65535];
 } dnbd3_binstring_t;
 // Do not always allocate as much memory as required to hold the entire binstring struct,
-// but only as much as is required to hold the actual data
+// but only as much as is required to hold the actual data (relevant for kernel module)
 #define NEW_BINSTRING(_name, _len) \
 	dnbd3_binstring_t *_name = malloc(sizeof(uint16_t) + _len); \
 	_name->len = _len
@@ -70,10 +70,11 @@ typedef struct
 typedef struct
 {
 	char comment[COMMENT_LENGTH];
-	time_t last_told;
+	time_t lastReached;
 	dnbd3_host_t host;
 	int rtt[SERVER_RTT_PROBES];
 	int rttIndex;
+	int isPrivate;
 } dnbd3_alt_server_t;
 
 typedef struct
@@ -95,6 +96,7 @@ struct _dnbd3_image
 	char *lower_name;      // relative path, all lowercase, minus revision ID
 	uint8_t *cache_map;    // cache map telling which parts are locally cached, NULL if complete
 	uint32_t *crc32;       // list of crc32 checksums for each 16MiB block in image
+	uint32_t masterCrc32;  // CRC-32 of the crc-32 list
 	dnbd3_connection_t *uplink; // pointer to a server connection
 	uint64_t filesize;     // size of image
 	int cacheFd;           // used to write to the image, in case it is relayed. ONLY USE FROM UPLINK THREAD!
@@ -145,8 +147,16 @@ extern int _serverPenalty;
  */
 extern int _clientPenalty;
 
+/**
+ * Is server shutting down?
+ */
 extern int _shutdown;
 
+/**
+ * Is server allowed to provide images in proxy mode?
+ */
+extern int _isProxy;
+
 void globals_loadConfig();
 
 #endif /* GLOBALS_H_ */
diff --git a/src/server/helper.c b/src/server/helper.c
index bc723e3..6c0e822 100644
--- a/src/server/helper.c
+++ b/src/server/helper.c
@@ -2,11 +2,9 @@
 #include <arpa/inet.h>
 #include <string.h>
 #include <stdlib.h>
-#include <fcntl.h>
 #include <assert.h>
-#include <sys/stat.h>
 #include <sys/prctl.h> // For thread names
-#include "../types.h"
+
 #include "../config.h"
 
 /**
@@ -125,55 +123,6 @@ void trim_right(char * const string)
 		*end-- = '\0';
 }
 
-int file_exists(char *file)
-{
-	int fd = open( file, O_RDONLY );
-	if ( fd < 0 ) return FALSE;
-	close( fd );
-	return TRUE;
-}
-
-int file_writable(char *file)
-{
-	int fd = open( file, O_WRONLY );
-	if ( fd >= 0 ) {
-		close( fd );
-		return TRUE;
-	}
-	fd = open( file, O_WRONLY | O_CREAT, 0600 );
-	if ( fd < 0 ) return FALSE;
-	close( fd );
-	unlink( file );
-	return TRUE;
-}
-
-int mkdir_p(const char* path)
-{
-	assert( path != NULL );
-	if ( *path == '\0' ) return TRUE;
-	char buffer[strlen( path ) + 1];
-	strcpy( buffer, path );
-	char *current = buffer;
-	char *slash;
-	while ( (slash = strchr( current, '/' )) != NULL ) {
-		*slash = '\0';
-		if ( *buffer != '\0' && mkdir( buffer, 0750 ) != 0 && errno != EEXIST ) return FALSE;
-		*slash = '/';
-		current = slash + 1;
-	}
-	if ( mkdir( buffer, 0750 ) != 0 && errno != EEXIST ) return FALSE;
-	return TRUE;
-}
-
-int file_alloc(int fd, uint64_t offset, uint64_t size)
-{
-	if ( fallocate( fd, 0, offset, size ) == 0 ) return TRUE; // fast way
-	if ( posix_fallocate( fd, offset, size ) == 0 ) return TRUE; // slow way
-	if ( lseek( fd, offset + size - 1, SEEK_SET ) != offset ) return FALSE; // dumb way
-	if ( write( fd, "", 1 ) != 1 ) return FALSE;
-	return TRUE;
-}
-
 void setThreadName(char *name)
 {
 	if ( strlen( name ) > 16 ) name[16] = '\0';
diff --git a/src/server/helper.h b/src/server/helper.h
index e939a66..eb82a00 100644
--- a/src/server/helper.h
+++ b/src/server/helper.h
@@ -16,10 +16,6 @@ char host_to_string(const dnbd3_host_t *host, char *target, size_t targetlen);
 void strtolower(char *string);
 void remove_trailing_slash(char *string);
 void trim_right(char * const string);
-int file_exists(char *file);
-int file_writable(char *file);
-int mkdir_p(const char* path);
-int file_alloc(int fd, uint64_t offset, uint64_t size);
 void setThreadName(char *name);
 
 static inline int is_same_server(const dnbd3_host_t * const a, const dnbd3_host_t * const b)
diff --git a/src/server/image.c b/src/server/image.c
index 10c650c..11029df 100644
--- a/src/server/image.c
+++ b/src/server/image.c
@@ -1,5 +1,6 @@
 #include "image.h"
 #include "helper.h"
+#include "fileutil.h"
 #include "memlog.h"
 #include "uplink.h"
 #include "locks.h"
@@ -7,6 +8,7 @@
 #include "protocol.h"
 #include "sockhelper.h"
 #include "altservers.h"
+#include "server.h"
 
 #include <assert.h>
 #include <stdio.h>
@@ -45,6 +47,7 @@ static int image_load_all_internal(char *base, char *path);
 static int image_try_load(char *base, char *path, int withUplink);
 static int64_t image_pad(const char *path, const int64_t currentSize);
 static int image_clone(int sock, char *name, uint16_t revision, uint64_t imageSize);
+static int image_ensureDiskSpace(uint64_t size);
 
 // ##########################################
 
@@ -239,15 +242,19 @@ dnbd3_image_t* image_get(char *name, uint16_t revision)
 
 	spin_lock( &candidate->lock );
 	spin_unlock( &_images_lock );
+	candidate->users++;
+	spin_unlock( &candidate->lock );
 
 	// Found, see if it works
 	struct stat st;
 	if ( candidate->working && stat( candidate->path, &st ) < 0 ) {
+		// Either the image is already marked as "not working", or the file cannot be accessed
 		printf( "[DEBUG] File '%s' has gone away...\n", candidate->path );
 		candidate->working = FALSE; // No file? OUT!
+	} else if ( !candidate->working && candidate->cache_map != NULL && file_isWritable( candidate->path ) ) {
+		// Not working and has file + cache-map, try to init uplink (uplink_init will check if proxy mode is enabled)
+		uplink_init( candidate, -1, NULL );
 	}
-	candidate->users++;
-	spin_unlock( &candidate->lock );
 	return candidate; // Success :-)
 }
 
@@ -259,6 +266,7 @@ dnbd3_image_t* image_get(char *name, uint16_t revision)
  */
 dnbd3_image_t* image_lock(dnbd3_image_t *image)
 {
+	if ( image == NULL ) return NULL ;
 	int i;
 	spin_lock( &_images_lock );
 	for (i = 0; i < _num_images; ++i) {
@@ -307,7 +315,6 @@ void image_release(dnbd3_image_t *image)
 	// Not found, free
 	spin_unlock( &image->lock );
 	spin_unlock( &_images_lock );
-	image_free( image );
 }
 
 /**
@@ -524,6 +531,7 @@ static int image_try_load(char *base, char *path, int withUplink)
 	if ( fileSize % DNBD3_BLOCK_SIZE != 0 ) {
 		memlogf( "[INFO] Image size of '%s' is not a multiple of %d, fixing...", path, (int)DNBD3_BLOCK_SIZE );
 		fileSize = image_pad( path, fileSize );
+		if ( fileSize == 0 ) goto load_error;
 	}
 	// 1. Allocate memory for the cache map if the image is incomplete
 	sprintf( mapFile, "%s.map", path );
@@ -544,6 +552,7 @@ static int image_try_load(char *base, char *path, int withUplink)
 	// but maybe later on you want better security
 	// 2. Load CRC-32 list of image
 	sprintf( hashFile, "%s.crc", path );
+	uint32_t masterCrc;
 	int fdHash = open( hashFile, O_RDONLY );
 	if ( fdHash >= 0 ) {
 		off_t fs = lseek( fdHash, 0, SEEK_END );
@@ -553,8 +562,7 @@ static int image_try_load(char *base, char *path, int withUplink)
 			if ( 0 != lseek( fdHash, 0, SEEK_SET ) ) {
 				memlogf( "[WARNING] Could not seek back to beginning of '%s'", hashFile );
 			} else {
-				uint32_t crcCrc;
-				if ( read( fdHash, &crcCrc, sizeof(crcCrc) ) != 4 ) {
+				if ( read( fdHash, &masterCrc, sizeof(masterCrc) ) != 4 ) {
 					memlogf( "[WARNING] Error reading first crc32 of '%s'", path );
 				} else {
 					crc32list = calloc( hashBlocks, sizeof(uint32_t) );
@@ -565,7 +573,7 @@ static int image_try_load(char *base, char *path, int withUplink)
 					} else {
 						uint32_t lists_crc = crc32( 0L, Z_NULL, 0 );
 						lists_crc = crc32( lists_crc, (Bytef*)crc32list, hashBlocks * sizeof(uint32_t) );
-						if ( lists_crc != crcCrc ) {
+						if ( lists_crc != masterCrc ) {
 							free( crc32list );
 							crc32list = NULL;
 							memlogf( "[WARNING] CRC-32 of CRC-32 list mismatch. CRC-32 list of '%s' might be corrupted.", path );
@@ -604,6 +612,7 @@ static int image_try_load(char *base, char *path, int withUplink)
 		} else if ( existing->crc32 == NULL && crc32list != NULL ) {
 			memlogf( "[INFO] Found CRC-32 list for already loaded image, adding...", path );
 			existing->crc32 = crc32list;
+			existing->masterCrc32 = masterCrc;
 			crc32list = NULL;
 			function_return = TRUE;
 			goto load_error;
@@ -622,6 +631,7 @@ static int image_try_load(char *base, char *path, int withUplink)
 	image->lower_name = strdup( imgName );
 	image->cache_map = cache_map;
 	image->crc32 = crc32list;
+	image->masterCrc32 = masterCrc;
 	image->uplink = NULL;
 	image->filesize = fileSize;
 	image->rid = revision;
@@ -707,7 +717,7 @@ int image_create(char *image, int revision, uint64_t size)
 		*lastSlash = '/';
 		snprintf( path, PATHLEN, "%s/%s.r%d", _basePath, image, revision );
 	}
-	if ( file_exists( path ) ) {
+	if ( file_isReadable( path ) ) {
 		memlogf( "[ERROR] Image %s with rid %d already exists!", image, revision );
 		return FALSE;
 	}
@@ -759,7 +769,7 @@ int image_create(char *image, int revision, uint64_t size)
  */
 dnbd3_image_t* image_getOrClone(char *name, uint16_t revision)
 {
-	// TODO: Simply return image_get if no authoritative servers are configured
+	if ( !_isProxy ) return image_get( name, revision );
 	int i;
 	const size_t len = strlen( name );
 	// Sanity check
@@ -813,6 +823,8 @@ dnbd3_image_t* image_getOrClone(char *name, uint16_t revision)
 		if ( remoteVersion < MIN_SUPPORTED_SERVER ) goto server_fail;
 		if ( revision != 0 && remoteRid != revision ) goto server_fail;
 		if ( remoteImageSize < DNBD3_BLOCK_SIZE || remoteName == NULL || strcmp( name, remoteName ) != 0 ) goto server_fail;
+		if ( remoteImageSize > SERVER_MAX_PROXY_IMAGE_SIZE ) goto server_fail;
+		if ( !image_ensureDiskSpace( remoteImageSize ) ) goto server_fail;
 		if ( !image_clone( sock, name, remoteRid, remoteImageSize ) ) goto server_fail;
 		// Cloning worked :-)
 		uplinkSock = sock;
@@ -849,20 +861,28 @@ static int image_clone(int sock, char *name, uint16_t revision, uint64_t imageSi
 	const size_t len = strlen( _basePath ) + strlen( name ) + 20;
 	char crcFile[len];
 	snprintf( crcFile, len, "%s/%s.r%d.crc", _basePath, name, (int)revision );
-	if ( !file_exists( crcFile ) ) {
+	if ( !file_isReadable( crcFile ) ) {
 		// Get crc32list from remote server
 		size_t crc32len = IMGSIZE_TO_HASHBLOCKS(imageSize) * sizeof(uint32_t);
-		uint8_t *crc32 = malloc( crc32len );
-		if ( !dnbd3_get_crc32( sock, crc32, &crc32len ) ) {
-			free( crc32 );
+		uint32_t masterCrc;
+		uint8_t *crc32list = malloc( crc32len );
+		if ( !dnbd3_get_crc32( sock, &masterCrc, crc32list, &crc32len ) ) {
+			free( crc32list );
 			return FALSE;
 		}
-		if ( crc32len > 0 ) {
-			int fd = open( crcFile, O_WRONLY | O_CREAT, 0640 );
-			write( fd, crc32, crc32len );
-			close( fd );
+		if ( crc32len != 0 ) {
+			uint32_t lists_crc = crc32( 0L, Z_NULL, 0 );
+			lists_crc = crc32( lists_crc, (Bytef*)crc32list, crc32len );
+			if ( lists_crc != masterCrc ) {
+				memlogf( "[WARNING] OTF-Clone: Corrupted CRC-32 list. ignored. (%s)", name );
+			} else {
+				int fd = open( crcFile, O_WRONLY | O_CREAT, 0640 );
+				write( fd, &lists_crc, sizeof(uint32_t) );
+				write( fd, crc32list, crc32len );
+				close( fd );
+			}
 		}
-		free( crc32 );
+		free( crc32list );
 	}
 	// HACK: Chop of ".crc" to get the image file name
 	crcFile[strlen( crcFile ) - 4] = '\0';
@@ -918,7 +938,7 @@ int image_generateCrcFile(char *image)
 		close( fdImage );
 		return FALSE;
 	}
-// CRC of all CRCs goes first. Don't know it yet, write 4 bytes dummy data.
+	// CRC of all CRCs goes first. Don't know it yet, write 4 bytes dummy data.
 	if ( write( fdCrc, crcFile, 4 ) != 4 ) {
 		printf( "Write error\n" );
 		close( fdImage );
@@ -969,7 +989,7 @@ int image_generateCrcFile(char *image)
 	close( fdImage );
 	printf( "done!\nGenerating master-crc..." );
 	fflush( stdout );
-// File is written - read again to calc master crc
+	// File is written - read again to calc master crc
 	if ( lseek( fdCrc, 4, SEEK_SET ) != 4 ) {
 		printf( "Could not seek to beginning of crc list in file\n" );
 		close( fdCrc );
@@ -1056,8 +1076,65 @@ static int64_t image_pad(const char *path, const int64_t currentSize)
 	if ( success ) {
 		return currentSize + missing;
 	} else {
-		return currentSize - (DNBD3_BLOCK_SIZE - missing);
+		return 0;
+	}
+}
+
+/**
+ * Make sure at least size bytes are available in _basePath.
+ * Will delete old images to make room for new ones.
+ * TODO: Store last access time of images. Currently the
+ * last access time is reset on server restart. Thus it will
+ * currently only delete images if server uptime is > 10 hours
+ * Return TRUE iff enough space is available. FALSE in random other cases
+ */
+static int image_ensureDiskSpace(uint64_t size)
+{
+	for (;;) {
+		const uint64_t available = file_freeDiskSpace( _basePath );
+		if ( available > size ) return TRUE;
+		if ( dnbd3_serverUptime() < 10 * 3600 ) {
+			memlogf( "[INFO] Only %dMiB free, %dMiB requested, but server uptime < 10 hours...", (int)(available / (1024 * 1024)),
+			        (int)(size / (1024 * 1024)) );
+			return FALSE;
+		}
+		memlogf( "[INFO] Only %dMiB free, %dMiB requested, freeing an image...", (int)(available / (1024 * 1024)),
+		        (int)(size / (1024 * 1024)) );
+		// Find least recently used image
+		dnbd3_image_t *oldest = NULL;
+		time_t mtime = 0;
+		int i;
+		spin_lock( &_images_lock );
+		for (i = 0; i < _num_images; ++i) {
+			if ( _images[i] == NULL ) continue;
+			dnbd3_image_t *current = image_lock( _images[i] );
+			if ( current == NULL ) continue;
+			if ( current->users == 1 ) { // Just from the lock above
+				if ( oldest == NULL || oldest->atime > _images[i]->atime ) {
+					// Oldest access time so far
+					oldest = _images[i];
+					if ( oldest->atime == 0 ) mtime = file_lastModification( oldest->path );
+				} else if ( oldest->atime == 0 && _images[i]->atime == 0 ) {
+					// Oldest access time is 0 (=never used since server startup), so take file modification time into account
+					const time_t m = file_lastModification( _images[i]->path );
+					if ( m < mtime ) {
+						mtime = m;
+						oldest = _images[i];
+					}
+				}
+			}
+			image_release( current );
+		}
+		spin_unlock( &_images_lock );
+		if ( oldest == NULL ) return FALSE;
+		oldest = image_lock( oldest );
+		if ( oldest == NULL ) return FALSE;
+		memlogf( "[INFO] '%s' has to go!", oldest->lower_name );
+		unlink( oldest->path );
+		image_remove( oldest );
+		image_release( oldest );
 	}
+	return FALSE;
 }
 
 /*
diff --git a/src/server/net.c b/src/server/net.c
index 4e25564..cd38205 100644
--- a/src/server/net.c
+++ b/src/server/net.c
@@ -329,10 +329,13 @@ void *net_client_handler(void *dnbd3_client)
 				reply.cmd = CMD_GET_CRC32;
 				if ( image->crc32 == NULL ) {
 					reply.size = 0;
+					send_reply( client->sock, &reply, NULL );
 				} else {
-					reply.size = IMGSIZE_TO_HASHBLOCKS(image->filesize) * 4;
+					const int size = reply.size = (IMGSIZE_TO_HASHBLOCKS(image->filesize) + 1) * sizeof(uint32_t);
+					send_reply( client->sock, &reply, NULL );
+					send( client->sock, &image->masterCrc32, sizeof(uint32_t), 0 );
+					send( client->sock, image->crc32, size - sizeof(uint32_t), 0 );
 				}
-				send_reply( client->sock, &reply, image->crc32 );
 				break;
 
 			default:
diff --git a/src/server/protocol.h b/src/server/protocol.h
index 96856f8..c8010ab 100644
--- a/src/server/protocol.h
+++ b/src/server/protocol.h
@@ -54,7 +54,7 @@ static inline int dnbd3_get_block(int sock, uint64_t offset, uint32_t size)
 	return send( sock, &request, sizeof(request), 0 ) == sizeof(request);
 }
 
-static inline int dnbd3_get_crc32(int sock, uint8_t *buffer, size_t *bufferLen)
+static inline int dnbd3_get_crc32(int sock, uint32_t *master, void *buffer, size_t *bufferLen)
 {
 	dnbd3_request_t request;
 	dnbd3_reply_t reply;
@@ -66,10 +66,16 @@ static inline int dnbd3_get_crc32(int sock, uint8_t *buffer, size_t *bufferLen)
 	fixup_request( request );
 	if ( send( sock, &request, sizeof(request), 0 ) != sizeof(request) ) return FALSE;
 	if ( !dnbd3_get_reply( sock, &reply ) ) return FALSE;
+	if ( reply.size == 0 ) {
+		*bufferLen = 0;
+		return TRUE;
+	}
+	if ( reply.size < 4 ) return FALSE;
+	reply.size -= 4;
 	if ( reply.cmd != CMD_GET_CRC32 || reply.size > *bufferLen ) return FALSE;
 	*bufferLen = reply.size;
-	if ( reply.size == 0 ) return TRUE;
-	return recv( sock, buffer, reply.size, 0 ) == (int)reply.size;
+	return recv( sock, master, sizeof(uint32_t), 0 ) == sizeof(uint32_t)
+	        && recv( sock, buffer, reply.size, 0 ) == (int)reply.size;
 }
 
 /**
diff --git a/src/server/server.c b/src/server/server.c
index 8252054..57e9b94 100644
--- a/src/server/server.c
+++ b/src/server/server.c
@@ -55,8 +55,10 @@ dnbd3_client_t *_clients[SERVER_MAX_CLIENTS];
 int _num_clients = 0;
 pthread_spinlock_t _clients_lock;
 
-char *_rpc_password = NULL;
-char *_cache_dir = NULL;
+/**
+ * Time the server was started
+ */
+static time_t _startupTime = 0;
 
 static int dnbd3_add_client(dnbd3_client_t *client);
 static void dnbd3_load_config();
@@ -301,6 +303,8 @@ int main(int argc, char *argv[])
 		return EXIT_FAILURE;
 	}
 
+	_startupTime = time( NULL );
+
 	// setup network
 	sockets[socket_count] = sock_listen_any( PF_INET, PORT );
 	if ( sockets[socket_count] != -1 ) ++socket_count;
@@ -484,3 +488,8 @@ void dnbd3_handle_sigusr1(int signum)
 	memlogf( "INFO: SIGUSR1 (%s) received, re-scanning image directory", strsignal( signum ) );
 	image_loadAll( NULL );
 }
+
+int dnbd3_serverUptime()
+{
+	return (int)(time( NULL ) - _startupTime);
+}
diff --git a/src/server/server.h b/src/server/server.h
index ba35bf7..f45ae98 100644
--- a/src/server/server.h
+++ b/src/server/server.h
@@ -34,8 +34,6 @@ extern dnbd3_client_t *_clients[SERVER_MAX_CLIENTS];
 extern int _num_clients;
 extern pthread_spinlock_t _clients_lock;
 
-extern char *_rpc_password, *_cache_dir;
-
 #ifdef _DEBUG
 extern int _fake_delay;
 #endif
@@ -44,6 +42,7 @@ void dnbd3_cleanup();
 void dnbd3_remove_client(dnbd3_client_t *client);
 dnbd3_client_t* dnbd3_free_client(dnbd3_client_t *client);
 dnbd3_client_t* dnbd3_init_client(struct sockaddr_storage *client, int fd);
+int dnbd3_serverUptime();
 
 #if !defined(_FILE_OFFSET_BITS) || _FILE_OFFSET_BITS != 64
 #error Please set _FILE_OFFSET_BITS to 64 in your makefile/configuration
diff --git a/src/server/uplink.c b/src/server/uplink.c
index dc2874e..e4bf51f 100644
--- a/src/server/uplink.c
+++ b/src/server/uplink.c
@@ -6,6 +6,8 @@
 #include "helper.h"
 #include "altservers.h"
 #include "helper.h"
+#include "protocol.h"
+
 #include <pthread.h>
 #include <sys/socket.h>
 #include <string.h>
@@ -17,11 +19,13 @@
 #include <stdlib.h>
 #include <stdio.h>
 #include <inttypes.h>
+#include <zlib.h>
 
 static void* uplink_mainloop(void *data);
 static void uplink_send_requests(dnbd3_connection_t *link, int newOnly);
 static void uplink_handle_receive(dnbd3_connection_t *link);
 static int uplink_send_keepalive(const int fd);
+static void uplink_addCrc32(dnbd3_connection_t *uplink);
 
 // ############ uplink connection handling
 
@@ -32,6 +36,7 @@ static int uplink_send_keepalive(const int fd);
  */
 int uplink_init(dnbd3_image_t *image, int sock, dnbd3_host_t *host)
 {
+	if ( !_isProxy ) return FALSE;
 	dnbd3_connection_t *link = NULL;
 	assert( image != NULL );
 	spin_lock( &image->lock );
@@ -122,7 +127,8 @@ int uplink_request(dnbd3_client_t *client, uint64_t handle, uint64_t start, uint
 	int existingType = -1; // ULR_* type of existing request
 	int i;
 	int freeSlot = -1;
-	const uint64_t end = start + length;
+	if ( length < 1024 * 1024 ) length = 1024 * 1024;
+	const uint64_t end = MIN(start + length, uplink->image->filesize);
 
 	spin_lock( &uplink->queueLock );
 	spin_unlock( &client->image->lock );
@@ -193,19 +199,17 @@ static void* uplink_mainloop(void *data)
 		memlogf( "[WARNING] epoll_create failed. Uplink unavailable." );
 		goto cleanup;
 	}
-	{
-		link->signal = eventfd( 0, EFD_NONBLOCK );
-		if ( link->signal < 0 ) {
-			memlogf( "[WARNING] error creating pipe. Uplink unavailable." );
-			goto cleanup;
-		}
-		memset( &ev, 0, sizeof(ev) );
-		ev.events = EPOLLIN;
-		ev.data.fd = link->signal;
-		if ( epoll_ctl( fdEpoll, EPOLL_CTL_ADD, link->signal, &ev ) < 0 ) {
-			memlogf( "[WARNING] adding eventfd to epoll set failed" );
-			goto cleanup;
-		}
+	link->signal = eventfd( 0, EFD_NONBLOCK );
+	if ( link->signal < 0 ) {
+		memlogf( "[WARNING] error creating pipe. Uplink unavailable." );
+		goto cleanup;
+	}
+	memset( &ev, 0, sizeof(ev) );
+	ev.events = EPOLLIN;
+	ev.data.fd = link->signal;
+	if ( epoll_ctl( fdEpoll, EPOLL_CTL_ADD, link->signal, &ev ) < 0 ) {
+		memlogf( "[WARNING] adding eventfd to epoll set failed" );
+		goto cleanup;
 	}
 	while ( !_shutdown && !link->shutdown ) {
 		// Check if server switch is in order
@@ -216,6 +220,10 @@ static void* uplink_mainloop(void *data)
 			const int fd = link->fd;
 			link->fd = link->betterFd;
 			if ( fd != -1 ) close( fd );
+			// If we don't have a crc32 list yet, see if the new server has one
+			if ( link->image->crc32 == NULL ) {
+				uplink_addCrc32( link );
+			}
 			// Re-send all pending requests
 			uplink_send_requests( link, FALSE );
 			link->betterFd = -1;
@@ -275,7 +283,7 @@ static void* uplink_mainloop(void *data)
 				int ret;
 				do {
 					ret = read( link->signal, buffer, sizeof buffer );
-				} while ( ret > 0 ); // Throw data away, this is just used for waking this thread up
+				} while ( ret == sizeof buffer ); // Throw data away, this is just used for waking this thread up
 				if ( ret == 0 ) {
 					memlogf( "[WARNING] Eventfd of uplink for %s closed! Things will break!", link->image->lower_name );
 				}
@@ -296,7 +304,7 @@ static void* uplink_mainloop(void *data)
 			}
 		}
 		// Done handling epoll sockets
-		// See if we should trigger a RTT measurement
+		// See if we should trigger an RTT measurement
 		if ( link->rttTestResult == RTT_IDLE || link->rttTestResult == RTT_DONTCHANGE ) {
 			const time_t now = time( NULL );
 			if ( nextAltCheck - now > SERVER_RTT_DELAY_MAX ) {
@@ -515,3 +523,25 @@ static int uplink_send_keepalive(const int fd)
 	}
 	return send( fd, &request, sizeof(request), 0 ) == sizeof(request);
 }
+
+static void uplink_addCrc32(dnbd3_connection_t *uplink)
+{
+	dnbd3_image_t *image = uplink->image;
+	if ( image == NULL || image->filesize == 0 ) return;
+	size_t bytes = IMGSIZE_TO_HASHBLOCKS(image->filesize) * sizeof(uint32_t);
+	uint32_t masterCrc;
+	uint32_t *buffer = malloc( bytes );
+	if ( !dnbd3_get_crc32( uplink->fd, &masterCrc, &buffer, &bytes ) || bytes == 0 ) {
+		free( buffer );
+		return;
+	}
+	uint32_t lists_crc = crc32( 0L, Z_NULL, 0 );
+	lists_crc = crc32( lists_crc, (Bytef*)buffer, bytes );
+	if ( lists_crc != masterCrc ) {
+		memlogf( "[WARNING] Received corrupted crc32 list from uplink server (%s)!", uplink->image->lower_name );
+		free( buffer );
+		return;
+	}
+	uplink->image->masterCrc32 = masterCrc;
+	uplink->image->crc32 = buffer;
+}