[SERVER] Use O_DIRECT for integrity checks

The idea is that for full image checks, we don't want to
pollute the fs cache with gigabytes of data that won't be
needed again soon. This would certainly hurt performance
on servers that dont have hundreds of GBs of RAM.
For single block checks during replication this has the
advantage that we don't check the block in memory before
it hit the disk once, but actually flush the data to disk,
then remove it from the page cache, and only then read it
again, from disk.

TODO: Might be worth making this a config option

1 files changed, 12 insertions, 4 deletions

diff --git a/src/server/image.c b/src/server/image.c
index 673a269..9d36317 100644
--- a/src/server/image.c
+++ b/src/server/image.c
@@ -72,6 +72,8 @@ void image_updateCachemap(dnbd3_image_t *image, uint64_t start, uint64_t end, co
 	assert( image != NULL );
 	// This should always be block borders due to how the protocol works, but better be safe
 	// than accidentally mark blocks as cached when they really aren't entirely cached.
+	assert( end <= image->virtualFilesize );
+	assert( start <= end );
 	if ( set ) {
 		// If we set as cached, move "inwards" in case we're not at 4k border
 		end &= ~(uint64_t)(DNBD3_BLOCK_SIZE - 1);
@@ -81,6 +83,8 @@ void image_updateCachemap(dnbd3_image_t *image, uint64_t start, uint64_t end, co
 		start &= ~(uint64_t)(DNBD3_BLOCK_SIZE - 1);
 		end = (uint64_t)(end + DNBD3_BLOCK_SIZE - 1) & ~(uint64_t)(DNBD3_BLOCK_SIZE - 1);
 	}
+	if ( start >= end )
+		return;
 	bool setNewBlocks = false;
 	uint64_t pos = start;
 	spin_lock( &image->lock );
@@ -1601,7 +1605,10 @@ bool image_checkBlocksCrc32(const int fd, uint32_t *crc32list, const int *blocks
  */
 static bool image_calcBlockCrc32(const int fd, const size_t block, const uint64_t realFilesize, uint32_t *crc)
 {
-	char buffer[262144];
+	// Make buffer 4k aligned in case fd has O_DIRECT set
+#define BSIZE 262144
+	char rawBuffer[BSIZE + DNBD3_BLOCK_SIZE];
+	char * const buffer = (char*)( ( (uintptr_t)rawBuffer + ( DNBD3_BLOCK_SIZE - 1 ) ) & ~( DNBD3_BLOCK_SIZE - 1 ) );
 	// How many bytes to read from the input file
 	const uint64_t bytesFromFile = MIN( HASH_BLOCK_SIZE, realFilesize - ( block * HASH_BLOCK_SIZE) );
 	// Determine how many bytes we had to read if the file size were a multiple of 4k
@@ -1614,7 +1621,7 @@ static bool image_calcBlockCrc32(const int fd, const size_t block, const uint64_
 	*crc = crc32( 0, NULL, 0 );
 	// Calculate the crc32 by reading data from the file
 	while ( bytes < bytesFromFile ) {
-		const size_t n = (size_t)MIN( sizeof(buffer), bytesFromFile - bytes );
+		const size_t n = (size_t)MIN( BSIZE, bytesFromFile - bytes );
 		const ssize_t r = pread( fd, buffer, n, readPos + bytes );
 		if ( r <= 0 ) {
 			logadd( LOG_WARNING, "CRC: Read error (errno=%d)", errno );
@@ -1625,16 +1632,17 @@ static bool image_calcBlockCrc32(const int fd, const size_t block, const uint64_
 	}
 	// If the virtual file size is different, keep going using nullbytes
 	if ( bytesFromFile < virtualBytesFromFile ) {
-		memset( buffer, 0, sizeof(buffer) );
+		memset( buffer, 0, BSIZE );
 		bytes = (size_t)( virtualBytesFromFile - bytesFromFile );
 		while ( bytes != 0 ) {
-			const size_t len = MIN( sizeof(buffer), bytes );
+			const size_t len = MIN( BSIZE, bytes );
 			*crc = crc32( *crc, (uint8_t*)buffer, len );
 			bytes -= len;
 		}
 	}
 	*crc = net_order_32( *crc );
 	return true;
+#undef BSIZE
 }
 
 /**