Implement CRC-32 list generation; fix quick CRC-32 check on image loading

7 files changed, 150 insertions, 27 deletions

diff --git a/CMakeLists.txt b/CMakeLists.txt
index 3451816..aaba077 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -6,8 +6,8 @@ PROJECT(dnbd3)
 CMAKE_MINIMUM_REQUIRED(VERSION 2.8.0)
 
 SET(CMAKE_BUILD_TYPE Debug)
-SET(CMAKE_C_FLAGS_DEBUG "-std=c99 -O0 -g -Wall -Wno-unused-result -D_GNU_SOURCE -D_DEBUG")
-SET(CMAKE_C_FLAGS_RELEASE "-std=c99 -O2 -Wno-unused-result -D_GNU_SOURCE")
+SET(CMAKE_C_FLAGS_DEBUG "-std=c99 -O0 -g -Wall -Wno-unused-result -D_GNU_SOURCE -D_DEBUG -Wno-multichar")
+SET(CMAKE_C_FLAGS_RELEASE "-std=c99 -O2 -Wno-unused-result -D_GNU_SOURCE -Wno-multichar")
 SET(CMAKE_CXX_FLAGS_DEBUG "-std=c99 -O0 -g -Wall -Wno-unused-result -D_GNU_SOURCE -D_DEBUG")
 SET(CMAKE_CXX_FLAGS_RELEASE "-std=c99 -O2 -Wno-unused-result -D_GNU_SOURCE" )
 
diff --git a/src/server/helper.h b/src/server/helper.h
index f0285a2..a9a8e79 100644
--- a/src/server/helper.h
+++ b/src/server/helper.h
@@ -6,6 +6,7 @@
 #include <string.h>
 #include <errno.h>
 #include <unistd.h>
+#include "../types.h"
 
 char parse_address(char *string, dnbd3_host_t *host);
 char host_to_string(const dnbd3_host_t *host, char *target, size_t targetlen);
@@ -101,7 +102,7 @@ static inline int strend(char *string, char *suffix)
 #define IMGSIZE_TO_MAPBYTES(bytes) ((int)(((bytes) + (1 << 15) - 1) >> 15))
 
 // calculate number of hash blocks in file. One hash block is 16MiB
-#define HASH_BLOCK_SIZE (1 << 24)
+#define HASH_BLOCK_SIZE ((int64_t)(1 << 24))
 #define IMGSIZE_TO_HASHBLOCKS(bytes) ((int)(((bytes) + HASH_BLOCK_SIZE - 1) / HASH_BLOCK_SIZE))
 
 #endif
diff --git a/src/server/image.c b/src/server/image.c
index 3e24b86..660186c 100644
--- a/src/server/image.c
+++ b/src/server/image.c
@@ -76,7 +76,7 @@ int image_save_cache_map(dnbd3_image_t *image)
 	strcpy( mapfile, image->path );
 	strcat( mapfile, ".map" );
 
-	fd = open( mapfile, O_WRONLY | O_CREAT, S_IRUSR | S_IWUSR );
+	fd = open( mapfile, O_WRONLY | O_CREAT, 0640 );
 	if ( fd < 0 ) return FALSE;
 
 	write( fd, image->cache_map, ((image->filesize + (1 << 15) - 1) >> 15) * sizeof(char) );
@@ -387,7 +387,14 @@ static int image_try_load(char *base, char *path)
 	}
 	// Check CRC32
 	if ( crc32list != NULL ) {
-		int blocks[] = { 0, rand() % hashBlocks, rand() % hashBlocks, -1 };
+		// This checks the first block and two random blocks (which might accidentally be the same)
+		// for corruption via the known crc32 list. This is very sloppy and is merely supposed
+		// to detect accidental corruption due to broken dnbd3-proxy functionality or file system
+		// corruption. If the image size is not a multiple of the hash block size, do not take the
+		// last block into consideration. It would always fail.
+		int blcks = hashBlocks;
+		if ( fileSize % HASH_BLOCK_SIZE != 0 ) blcks--;
+		int blocks[] = { 0, rand() % blcks, rand() % blcks, -1 };
 		if ( !image_check_blocks_crc32( fdImage, crc32list, blocks ) ) {
 			memlogf( "[ERROR] Quick integrity check for '%s' failed.", path );
 			goto load_error;
@@ -465,12 +472,120 @@ static int image_try_load(char *base, char *path)
 }
 
 /**
+ * Generate the crc32 block list file for the given file.
+ * This function wants a plain file name instead of a dnbd3_image_t,
+ * as it can be used directly from the command line.
+ */
+int image_generate_crc_file(char *image)
+{
+	int fdImage = open( image, O_RDONLY );
+	if ( fdImage < 0 ) {
+		printf( "Could not open %s.\n", image );
+		return FALSE;
+	}
+	char crcFile[strlen( image ) + 4 + 1];
+	sprintf( crcFile, "%s.crc", image );
+	struct stat sst;
+	if ( stat( crcFile, &sst ) == 0 ) {
+		printf( "CRC File for %s already exists! Delete it first if you want to regen.\n", image );
+		return FALSE;
+	}
+	int fdCrc = open( crcFile, O_RDWR | O_CREAT, 0640 );
+	if ( fdCrc < 0 ) {
+		printf( "Could not open CRC File %s for writing..\n", crcFile );
+		close( fdImage );
+		return FALSE;
+	}
+	// CRC of all CRCs goes first. Don't know it yet, write 4 bytes dummy data.
+	if ( write( fdCrc, crcFile, 4 ) != 4 ) {
+		printf( "Write error\n" );
+		close( fdImage );
+		close( fdCrc );
+		return FALSE;
+	}
+	char buffer[80000]; // Read buffer from image
+	int finished = FALSE; // end of file reached
+	int hasSum; // unwritten (unfinished?) crc32 exists
+	int blocksToGo = 0; // Count number of checksums written
+	printf( "Generating CRC32" );
+	fflush( stdout );
+	do {
+		// Start of a block - init
+		uint32_t crc = crc32( 0L, Z_NULL, 0 );
+		int remaining = HASH_BLOCK_SIZE;
+		hasSum = FALSE;
+		while ( remaining > 0 ) {
+			const int blockSize = MIN(remaining, sizeof(buffer));
+			const int ret = read( fdImage, buffer, blockSize );
+			if ( ret < 0 ) { // Error
+				printf( "Read error\n" );
+				close( fdImage );
+				close( fdCrc );
+				return FALSE;
+			} else if ( ret == 0 ) { // EOF
+				finished = TRUE;
+				break;
+			} else { // Read something
+				hasSum = TRUE;
+				crc = crc32( crc, (Bytef*)buffer, ret );
+				remaining -= ret;
+			}
+		}
+		// Write to file
+		if ( hasSum ) {
+			if ( write( fdCrc, &crc, 4 ) != 4 ) {
+				printf( "Write error\n" );
+				close( fdImage );
+				close( fdCrc );
+				return FALSE;
+			}
+			printf( "." );
+			fflush( stdout );
+			blocksToGo++;
+		}
+	} while ( !finished );
+	close( fdImage );
+	printf( "done!\nGenerating master-crc..." );
+	fflush( stdout );
+	// File is written - read again to calc master crc
+	if ( lseek( fdCrc, 4, SEEK_SET ) != 4 ) {
+		printf( "Could not seek to beginning of crc list in file\n" );
+		close( fdCrc );
+		return FALSE;
+	}
+	uint32_t crc = crc32( 0L, Z_NULL, 0 );
+	while ( blocksToGo > 0 ) {
+		const int numBlocks = MIN(1000, blocksToGo);
+		if ( read( fdCrc, buffer, numBlocks * 4 ) != numBlocks * 4 ) {
+			printf( "Could not re-read from crc32 file\n" );
+			close( fdCrc );
+			return FALSE;
+		}
+		crc = crc32( crc, (Bytef*)buffer, numBlocks * 4 );
+		blocksToGo -= numBlocks;
+	}
+	if ( lseek( fdCrc, 0, SEEK_SET ) != 0 ) {
+		printf( "Could not seek back to beginning of crc32 file\n" );
+		close( fdCrc );
+		return FALSE;
+	}
+	if ( write( fdCrc, &crc, 4 ) != 4 ) {
+		printf( "Could not write master crc to file\n" );
+		close( fdCrc );
+		return FALSE;
+	}
+	printf( "..done!\nCRC-32 file successfully generated.\n" );
+	fflush( stdout );
+	return TRUE;
+}
+
+/**
  * Check the CRC-32 of the given blocks. The array blocks is of variable length.
  * !! pass -1 as the last block so the function knows when to stop !!
  */
 static int image_check_blocks_crc32(int fd, uint32_t *crc32list, int *blocks)
 {
-	char buffer[32768];
+	char buffer[40000];
 	while ( *blocks != -1 ) {
 		if ( lseek( fd, *blocks * HASH_BLOCK_SIZE, SEEK_SET ) != *blocks * HASH_BLOCK_SIZE) {
 			memlogf( "Seek error" );
@@ -488,7 +603,10 @@ static int image_check_blocks_crc32(int fd, uint32_t *crc32list, int *blocks)
 			crc = crc32( crc, (Bytef*)buffer, r );
 			bytes += r;
 		}
-		if ( crc != crc32list[*blocks] ) return FALSE;
+		if ( crc != crc32list[*blocks] ) {
+			printf( "Block %d is %x, should be %x\n", *blocks, crc, crc32list[*blocks] );
+			return FALSE;
+		}
 		blocks++;
 	}
 	return TRUE;
diff --git a/src/server/image.h b/src/server/image.h
index d0b482c..7a4e72e 100644
--- a/src/server/image.h
+++ b/src/server/image.h
@@ -21,5 +21,6 @@ dnbd3_image_t* image_free(dnbd3_image_t *image);
 int image_load_all(char *path);
 
 
+int image_generate_crc_file(char *image);
 
 #endif
diff --git a/src/server/locks.c b/src/server/locks.c
index 9e8d4ad..d9eed35 100644
--- a/src/server/locks.c
+++ b/src/server/locks.c
@@ -242,6 +242,7 @@ void debug_locks_stop_watchdog()
 {
 #ifdef _DEBUG
 	_shutdown = TRUE;
+	printf("Killing debug watchdog...\n");
 	pthread_spin_lock( &initdestory );
 	pthread_spin_unlock( &initdestory );
 	pthread_join( watchdog, NULL );
diff --git a/src/server/net.c b/src/server/net.c
index 7fab6b9..ac82116 100644
--- a/src/server/net.c
+++ b/src/server/net.c
@@ -46,43 +46,43 @@ static inline char recv_request_header(int sock, dnbd3_request_t *request)
 	if ( (ret = recv( sock, request, sizeof(*request), MSG_WAITALL )) != sizeof(*request) ) {
 		if ( ret == 0 ) return 0;
 		printf( "[DEBUG] Error receiving request: Could not read message header (%d/%d)\n", ret, (int)sizeof(*request) );
-		return 0;
+		return FALSE;
 	}
 	// Make sure all bytes are in the right order (endianness)
 	fixup_request( *request );
 	if ( request->magic != dnbd3_packet_magic ) {
 		printf( "[DEBUG] Magic in client request incorrect (cmd: %d, len: %d)\n", (int)request->cmd, (int)request->size );
-		return 0;
+		return FALSE;
 	}
 	// Payload sanity check
 	if ( request->cmd != CMD_GET_BLOCK && request->size > MAX_PAYLOAD ) {
 		memlogf( "[WARNING] Client tries to send a packet of type %d with %d bytes payload. Dropping client.", (int)request->cmd,
 		        (int)request->size );
-		return 0;
+		return FALSE;
 	}
 #ifdef _DEBUG
 	if (_fake_delay) usleep(_fake_delay);
 #endif
-	return 1;
+	return TRUE;
 }
 
 static inline char recv_request_payload(int sock, uint32_t size, serialized_buffer_t *payload)
 {
 	if ( size == 0 ) {
 		memlogf( "[BUG] Called recv_request_payload() to receive 0 bytes" );
-		return 0;
+		return FALSE;
 	}
 	if ( size > MAX_PAYLOAD ) {
 		memlogf( "[BUG] Called recv_request_payload() for more bytes than the passed buffer could hold!" );
-		return 0;
+		return FALSE;
 	}
 	if ( recv( sock, payload->buffer, size, MSG_WAITALL ) != size ) {
 		printf( "[ERROR] Could not receive request payload of length %d\n", (int)size );
-		return 0;
+		return FALSE;
 	}
 	// Prepare payload buffer for reading
 	serializer_reset_read( payload, size );
-	return 1;
+	return TRUE;
 }
 
 static inline char send_reply(int sock, dnbd3_reply_t *reply, void *payload)
@@ -92,7 +92,7 @@ static inline char send_reply(int sock, dnbd3_reply_t *reply, void *payload)
 	if ( !payload || size == 0 ) {
 		if ( send( sock, reply, sizeof(dnbd3_reply_t), MSG_WAITALL ) != sizeof(dnbd3_reply_t) ) {
 			printf( "[DEBUG] Send failed (header-only)\n" );
-			return 0;
+			return FALSE;
 		}
 	} else {
 		struct iovec iov[2];
@@ -102,10 +102,10 @@ static inline char send_reply(int sock, dnbd3_reply_t *reply, void *payload)
 		iov[1].iov_len = size;
 		if ( writev( sock, iov, 2 ) != sizeof(dnbd3_reply_t) + size ) {
 			printf( "[DEBUG] Send failed (reply with payload of %u bytes)\n", size );
-			return 0;
+			return FALSE;
 		}
 	}
-	return 1;
+	return TRUE;
 }
 
 void *net_client_handler(void *dnbd3_client)
diff --git a/src/server/server.c b/src/server/server.c
index 56e5123..d48bb25 100644
--- a/src/server/server.c
+++ b/src/server/server.c
@@ -75,7 +75,9 @@ void dnbd3_print_help(char *argv_0)
 	printf( "-s or --stop        Stop running dnbd3-server\n" );
 	printf( "-i or --info        Print connected clients and used images\n" );
 	printf( "-H or --help        Show this help text and quit\n" );
-	printf( "-V or --version     Show version and quit\n" );
+	printf( "-v or --version     Show version and quit\n" );
+	printf( "Management functions:\n" );
+	printf( "--crc [image]       Generate crc block list for given image\n" );
 	exit( 0 );
 }
 
@@ -96,6 +98,7 @@ void dnbd3_cleanup()
 	int i;
 
 	_shutdown = TRUE;
+	debug_locks_stop_watchdog();
 	memlogf( "INFO: Cleanup...\n" );
 
 	for (int i = 0; i < socket_count; ++i) {
@@ -129,8 +132,6 @@ void dnbd3_cleanup()
 	_num_images = 0;
 	spin_unlock( &_images_lock );
 
-	debug_locks_stop_watchdog();
-
 	exit( EXIT_SUCCESS );
 }
 
@@ -145,7 +146,8 @@ int main(int argc, char *argv[])
 	static const char *optString = "f:d:nrsiHV?";
 	static const struct option longOpts[] = { { "file", required_argument, NULL, 'f' }, { "delay", required_argument, NULL, 'd' }, {
 	        "nodaemon", no_argument, NULL, 'n' }, { "reload", no_argument, NULL, 'r' }, { "stop", no_argument, NULL, 's' }, { "info",
-	        no_argument, NULL, 'i' }, { "help", no_argument, NULL, 'H' }, { "version", no_argument, NULL, 'V' } };
+	        no_argument, NULL, 'i' }, { "help", no_argument, NULL, 'H' }, { "version", no_argument, NULL, 'v' }, { "crc", required_argument,
+	        NULL, 'crc4' }, { 0, 0, 0, 0 } };
 
 	opt = getopt_long( argc, argv, optString, longOpts, &longIndex );
 
@@ -177,14 +179,14 @@ int main(int argc, char *argv[])
 			//dnbd3_rpc_send(RPC_IMG_LIST);
 			return EXIT_SUCCESS;
 		case 'H':
+		case '?':
 			dnbd3_print_help( argv[0] );
 			break;
-		case 'V':
+		case 'v':
 			dnbd3_print_version();
 			break;
-		case '?':
-			dnbd3_print_help( argv[0] );
-			break;
+		case 'crc4':
+			return image_generate_crc_file( optarg ) ? 0 : EXIT_FAILURE;
 		}
 		opt = getopt_long( argc, argv, optString, longOpts, &longIndex );
 	}
@@ -244,7 +246,7 @@ int main(int argc, char *argv[])
 
 	memlogf( "[INFO] Server is ready..." );
 
-	// main loop
+	// +++++++++++++++++++++++++++++++++++++++++++++++++++ main loop
 	while ( 1 ) {
 		len = sizeof(client);
 		fd = accept_any( sockets, socket_count, &client, &len );