From 35a48c0a39584d7783b6fcfd62354aedecf76e71 Mon Sep 17 00:00:00 2001 From: sr Date: Tue, 23 Jul 2013 15:26:11 +0200 Subject: Implement CRC-32 list generation; fix quick CRC-32 check on image loading --- src/server/helper.h | 3 +- src/server/image.c | 126 ++++++++++++++++++++++++++++++++++++++++++++++++++-- src/server/image.h | 1 + src/server/locks.c | 1 + src/server/net.c | 22 ++++----- src/server/server.c | 20 +++++---- 6 files changed, 148 insertions(+), 25 deletions(-) (limited to 'src') diff --git a/src/server/helper.h b/src/server/helper.h index f0285a2..a9a8e79 100644 --- a/src/server/helper.h +++ b/src/server/helper.h @@ -6,6 +6,7 @@ #include #include #include +#include "../types.h" char parse_address(char *string, dnbd3_host_t *host); char host_to_string(const dnbd3_host_t *host, char *target, size_t targetlen); @@ -101,7 +102,7 @@ static inline int strend(char *string, char *suffix) #define IMGSIZE_TO_MAPBYTES(bytes) ((int)(((bytes) + (1 << 15) - 1) >> 15)) // calculate number of hash blocks in file. One hash block is 16MiB -#define HASH_BLOCK_SIZE (1 << 24) +#define HASH_BLOCK_SIZE ((int64_t)(1 << 24)) #define IMGSIZE_TO_HASHBLOCKS(bytes) ((int)(((bytes) + HASH_BLOCK_SIZE - 1) / HASH_BLOCK_SIZE)) #endif diff --git a/src/server/image.c b/src/server/image.c index 3e24b86..660186c 100644 --- a/src/server/image.c +++ b/src/server/image.c @@ -76,7 +76,7 @@ int image_save_cache_map(dnbd3_image_t *image) strcpy( mapfile, image->path ); strcat( mapfile, ".map" ); - fd = open( mapfile, O_WRONLY | O_CREAT, S_IRUSR | S_IWUSR ); + fd = open( mapfile, O_WRONLY | O_CREAT, 0640 ); if ( fd < 0 ) return FALSE; write( fd, image->cache_map, ((image->filesize + (1 << 15) - 1) >> 15) * sizeof(char) ); @@ -387,7 +387,14 @@ static int image_try_load(char *base, char *path) } // Check CRC32 if ( crc32list != NULL ) { - int blocks[] = { 0, rand() % hashBlocks, rand() % hashBlocks, -1 }; + // This checks the first block and two random blocks (which might accidentally be the same) + // for corruption via the known crc32 list. This is very sloppy and is merely supposed + // to detect accidental corruption due to broken dnbd3-proxy functionality or file system + // corruption. If the image size is not a multiple of the hash block size, do not take the + // last block into consideration. It would always fail. + int blcks = hashBlocks; + if ( fileSize % HASH_BLOCK_SIZE != 0 ) blcks--; + int blocks[] = { 0, rand() % blcks, rand() % blcks, -1 }; if ( !image_check_blocks_crc32( fdImage, crc32list, blocks ) ) { memlogf( "[ERROR] Quick integrity check for '%s' failed.", path ); goto load_error; @@ -464,13 +471,121 @@ static int image_try_load(char *base, char *path) return function_return; } +/** + * Generate the crc32 block list file for the given file. + * This function wants a plain file name instead of a dnbd3_image_t, + * as it can be used directly from the command line. + */ +int image_generate_crc_file(char *image) +{ + int fdImage = open( image, O_RDONLY ); + if ( fdImage < 0 ) { + printf( "Could not open %s.\n", image ); + return FALSE; + } + char crcFile[strlen( image ) + 4 + 1]; + sprintf( crcFile, "%s.crc", image ); + struct stat sst; + if ( stat( crcFile, &sst ) == 0 ) { + printf( "CRC File for %s already exists! Delete it first if you want to regen.\n", image ); + return FALSE; + } + int fdCrc = open( crcFile, O_RDWR | O_CREAT, 0640 ); + if ( fdCrc < 0 ) { + printf( "Could not open CRC File %s for writing..\n", crcFile ); + close( fdImage ); + return FALSE; + } + // CRC of all CRCs goes first. Don't know it yet, write 4 bytes dummy data. + if ( write( fdCrc, crcFile, 4 ) != 4 ) { + printf( "Write error\n" ); + close( fdImage ); + close( fdCrc ); + return FALSE; + } + char buffer[80000]; // Read buffer from image + int finished = FALSE; // end of file reached + int hasSum; // unwritten (unfinished?) crc32 exists + int blocksToGo = 0; // Count number of checksums written + printf( "Generating CRC32" ); + fflush( stdout ); + do { + // Start of a block - init + uint32_t crc = crc32( 0L, Z_NULL, 0 ); + int remaining = HASH_BLOCK_SIZE; + hasSum = FALSE; + while ( remaining > 0 ) { + const int blockSize = MIN(remaining, sizeof(buffer)); + const int ret = read( fdImage, buffer, blockSize ); + if ( ret < 0 ) { // Error + printf( "Read error\n" ); + close( fdImage ); + close( fdCrc ); + return FALSE; + } else if ( ret == 0 ) { // EOF + finished = TRUE; + break; + } else { // Read something + hasSum = TRUE; + crc = crc32( crc, (Bytef*)buffer, ret ); + remaining -= ret; + } + } + // Write to file + if ( hasSum ) { + if ( write( fdCrc, &crc, 4 ) != 4 ) { + printf( "Write error\n" ); + close( fdImage ); + close( fdCrc ); + return FALSE; + } + printf( "." ); + fflush( stdout ); + blocksToGo++; + } + } while ( !finished ); + close( fdImage ); + printf( "done!\nGenerating master-crc..." ); + fflush( stdout ); + // File is written - read again to calc master crc + if ( lseek( fdCrc, 4, SEEK_SET ) != 4 ) { + printf( "Could not seek to beginning of crc list in file\n" ); + close( fdCrc ); + return FALSE; + } + uint32_t crc = crc32( 0L, Z_NULL, 0 ); + while ( blocksToGo > 0 ) { + const int numBlocks = MIN(1000, blocksToGo); + if ( read( fdCrc, buffer, numBlocks * 4 ) != numBlocks * 4 ) { + printf( "Could not re-read from crc32 file\n" ); + close( fdCrc ); + return FALSE; + } + crc = crc32( crc, (Bytef*)buffer, numBlocks * 4 ); + blocksToGo -= numBlocks; + } + if ( lseek( fdCrc, 0, SEEK_SET ) != 0 ) { + printf( "Could not seek back to beginning of crc32 file\n" ); + close( fdCrc ); + return FALSE; + } + if ( write( fdCrc, &crc, 4 ) != 4 ) { + printf( "Could not write master crc to file\n" ); + close( fdCrc ); + return FALSE; + } + printf( "..done!\nCRC-32 file successfully generated.\n" ); + fflush( stdout ); + return TRUE; +} + /** * Check the CRC-32 of the given blocks. The array blocks is of variable length. * !! pass -1 as the last block so the function knows when to stop !! */ static int image_check_blocks_crc32(int fd, uint32_t *crc32list, int *blocks) { - char buffer[32768]; + char buffer[40000]; while ( *blocks != -1 ) { if ( lseek( fd, *blocks * HASH_BLOCK_SIZE, SEEK_SET ) != *blocks * HASH_BLOCK_SIZE) { memlogf( "Seek error" ); @@ -488,7 +603,10 @@ static int image_check_blocks_crc32(int fd, uint32_t *crc32list, int *blocks) crc = crc32( crc, (Bytef*)buffer, r ); bytes += r; } - if ( crc != crc32list[*blocks] ) return FALSE; + if ( crc != crc32list[*blocks] ) { + printf( "Block %d is %x, should be %x\n", *blocks, crc, crc32list[*blocks] ); + return FALSE; + } blocks++; } return TRUE; diff --git a/src/server/image.h b/src/server/image.h index d0b482c..7a4e72e 100644 --- a/src/server/image.h +++ b/src/server/image.h @@ -21,5 +21,6 @@ dnbd3_image_t* image_free(dnbd3_image_t *image); int image_load_all(char *path); +int image_generate_crc_file(char *image); #endif diff --git a/src/server/locks.c b/src/server/locks.c index 9e8d4ad..d9eed35 100644 --- a/src/server/locks.c +++ b/src/server/locks.c @@ -242,6 +242,7 @@ void debug_locks_stop_watchdog() { #ifdef _DEBUG _shutdown = TRUE; + printf("Killing debug watchdog...\n"); pthread_spin_lock( &initdestory ); pthread_spin_unlock( &initdestory ); pthread_join( watchdog, NULL ); diff --git a/src/server/net.c b/src/server/net.c index 7fab6b9..ac82116 100644 --- a/src/server/net.c +++ b/src/server/net.c @@ -46,43 +46,43 @@ static inline char recv_request_header(int sock, dnbd3_request_t *request) if ( (ret = recv( sock, request, sizeof(*request), MSG_WAITALL )) != sizeof(*request) ) { if ( ret == 0 ) return 0; printf( "[DEBUG] Error receiving request: Could not read message header (%d/%d)\n", ret, (int)sizeof(*request) ); - return 0; + return FALSE; } // Make sure all bytes are in the right order (endianness) fixup_request( *request ); if ( request->magic != dnbd3_packet_magic ) { printf( "[DEBUG] Magic in client request incorrect (cmd: %d, len: %d)\n", (int)request->cmd, (int)request->size ); - return 0; + return FALSE; } // Payload sanity check if ( request->cmd != CMD_GET_BLOCK && request->size > MAX_PAYLOAD ) { memlogf( "[WARNING] Client tries to send a packet of type %d with %d bytes payload. Dropping client.", (int)request->cmd, (int)request->size ); - return 0; + return FALSE; } #ifdef _DEBUG if (_fake_delay) usleep(_fake_delay); #endif - return 1; + return TRUE; } static inline char recv_request_payload(int sock, uint32_t size, serialized_buffer_t *payload) { if ( size == 0 ) { memlogf( "[BUG] Called recv_request_payload() to receive 0 bytes" ); - return 0; + return FALSE; } if ( size > MAX_PAYLOAD ) { memlogf( "[BUG] Called recv_request_payload() for more bytes than the passed buffer could hold!" ); - return 0; + return FALSE; } if ( recv( sock, payload->buffer, size, MSG_WAITALL ) != size ) { printf( "[ERROR] Could not receive request payload of length %d\n", (int)size ); - return 0; + return FALSE; } // Prepare payload buffer for reading serializer_reset_read( payload, size ); - return 1; + return TRUE; } static inline char send_reply(int sock, dnbd3_reply_t *reply, void *payload) @@ -92,7 +92,7 @@ static inline char send_reply(int sock, dnbd3_reply_t *reply, void *payload) if ( !payload || size == 0 ) { if ( send( sock, reply, sizeof(dnbd3_reply_t), MSG_WAITALL ) != sizeof(dnbd3_reply_t) ) { printf( "[DEBUG] Send failed (header-only)\n" ); - return 0; + return FALSE; } } else { struct iovec iov[2]; @@ -102,10 +102,10 @@ static inline char send_reply(int sock, dnbd3_reply_t *reply, void *payload) iov[1].iov_len = size; if ( writev( sock, iov, 2 ) != sizeof(dnbd3_reply_t) + size ) { printf( "[DEBUG] Send failed (reply with payload of %u bytes)\n", size ); - return 0; + return FALSE; } } - return 1; + return TRUE; } void *net_client_handler(void *dnbd3_client) diff --git a/src/server/server.c b/src/server/server.c index 56e5123..d48bb25 100644 --- a/src/server/server.c +++ b/src/server/server.c @@ -75,7 +75,9 @@ void dnbd3_print_help(char *argv_0) printf( "-s or --stop Stop running dnbd3-server\n" ); printf( "-i or --info Print connected clients and used images\n" ); printf( "-H or --help Show this help text and quit\n" ); - printf( "-V or --version Show version and quit\n" ); + printf( "-v or --version Show version and quit\n" ); + printf( "Management functions:\n" ); + printf( "--crc [image] Generate crc block list for given image\n" ); exit( 0 ); } @@ -96,6 +98,7 @@ void dnbd3_cleanup() int i; _shutdown = TRUE; + debug_locks_stop_watchdog(); memlogf( "INFO: Cleanup...\n" ); for (int i = 0; i < socket_count; ++i) { @@ -129,8 +132,6 @@ void dnbd3_cleanup() _num_images = 0; spin_unlock( &_images_lock ); - debug_locks_stop_watchdog(); - exit( EXIT_SUCCESS ); } @@ -145,7 +146,8 @@ int main(int argc, char *argv[]) static const char *optString = "f:d:nrsiHV?"; static const struct option longOpts[] = { { "file", required_argument, NULL, 'f' }, { "delay", required_argument, NULL, 'd' }, { "nodaemon", no_argument, NULL, 'n' }, { "reload", no_argument, NULL, 'r' }, { "stop", no_argument, NULL, 's' }, { "info", - no_argument, NULL, 'i' }, { "help", no_argument, NULL, 'H' }, { "version", no_argument, NULL, 'V' } }; + no_argument, NULL, 'i' }, { "help", no_argument, NULL, 'H' }, { "version", no_argument, NULL, 'v' }, { "crc", required_argument, + NULL, 'crc4' }, { 0, 0, 0, 0 } }; opt = getopt_long( argc, argv, optString, longOpts, &longIndex ); @@ -177,14 +179,14 @@ int main(int argc, char *argv[]) //dnbd3_rpc_send(RPC_IMG_LIST); return EXIT_SUCCESS; case 'H': + case '?': dnbd3_print_help( argv[0] ); break; - case 'V': + case 'v': dnbd3_print_version(); break; - case '?': - dnbd3_print_help( argv[0] ); - break; + case 'crc4': + return image_generate_crc_file( optarg ) ? 0 : EXIT_FAILURE; } opt = getopt_long( argc, argv, optString, longOpts, &longIndex ); } @@ -244,7 +246,7 @@ int main(int argc, char *argv[]) memlogf( "[INFO] Server is ready..." ); - // main loop + // +++++++++++++++++++++++++++++++++++++++++++++++++++ main loop while ( 1 ) { len = sizeof(client); fd = accept_any( sockets, socket_count, &client, &len ); -- cgit v1.2.3-55-g7522