From faeb780fb33f79092f27b92dc5bbc13a8e7d6cbd Mon Sep 17 00:00:00 2001 From: Simon Rettberg Date: Fri, 30 Jan 2015 16:14:59 +0100 Subject: [SERVER] Use shared file handle for reading --- src/server/globals.h | 1 + src/server/image.c | 32 +++++++++++++++---------------- src/server/integrity.c | 16 ++++++---------- src/server/net.c | 51 ++++++++++++++++++++++++++------------------------ 4 files changed, 49 insertions(+), 51 deletions(-) diff --git a/src/server/globals.h b/src/server/globals.h index 2ea1c43..f9a1add 100644 --- a/src/server/globals.h +++ b/src/server/globals.h @@ -103,6 +103,7 @@ struct _dnbd3_image uint32_t masterCrc32; // CRC-32 of the crc-32 list dnbd3_connection_t * volatile uplink; // pointer to a server connection uint64_t filesize; // size of image + int readFd; // used to read the image. Used from multiple threads, so use atomic operations (pread et al) int cacheFd; // used to write to the image, in case it is relayed. ONLY USE FROM UPLINK THREAD! int rid; // revision of image int users; // clients currently using this image diff --git a/src/server/image.c b/src/server/image.c index 81c649d..ccee548 100644 --- a/src/server/image.c +++ b/src/server/image.c @@ -268,12 +268,7 @@ dnbd3_image_t* image_get(char *name, uint16_t revision, bool checkIfWorking) if ( !checkIfWorking ) return candidate; // Found, see if it works - struct stat st; - if ( candidate->working && stat( candidate->path, &st ) < 0 ) { - // Either the image is already marked as "not working", or the file cannot be accessed - printf( "[DEBUG] File '%s' has gone away...\n", candidate->path ); - candidate->working = false; // No file? OUT! - } else if ( !candidate->working && candidate->cache_map != NULL && candidate->uplink == NULL && file_isWritable( candidate->path ) ) { + if ( !candidate->working && candidate->cache_map != NULL && candidate->uplink == NULL && file_isWritable( candidate->path ) ) { // Not working and has file + cache-map, try to init uplink (uplink_init will check if proxy mode is enabled) uplink_init( candidate, -1, NULL ); } else if ( candidate->working && candidate->uplink != NULL && candidate->uplink->queueLen > SERVER_UPLINK_QUEUELEN_THRES ) { @@ -428,9 +423,8 @@ static dnbd3_image_t* image_free(dnbd3_image_t *image) free( image->path ); free( image->lower_name ); spin_unlock( &image->lock ); - if ( image->cacheFd != -1 ) { - close( image->cacheFd ); - } + if ( image->cacheFd != -1 ) close( image->cacheFd ); + if ( image->readFd != -1 ) close( image->readFd ); spin_destroy( &image->lock ); // memset( image, 0, sizeof(*image) ); @@ -613,6 +607,8 @@ static bool image_load(char *base, char *path, int withUplink) && memcmp( existing->crc32, crc32list, sizeof(uint32_t) * hashBlockCount ) != 0 ) { // Image will be replaced below memlogf( "[WARNING] CRC32 list of image '%s:%d' has changed.", existing->lower_name, (int)existing->rid ); + memlogf( "[WARNING] The image will be reloaded, but you should NOT replace existing images while the server is running." ); + memlogf( "[WARNING] Actually even if it's not running this should never be done. Use a new RID instead!" ); } else if ( existing->crc32 == NULL && crc32list != NULL ) { memlogf( "[INFO] Found CRC-32 list for already loaded image '%s:%d', adding...", existing->lower_name, (int)existing->rid ); existing->crc32 = crc32list; @@ -647,6 +643,7 @@ static bool image_load(char *base, char *path, int withUplink) image->filesize = fileSize; image->rid = revision; image->users = 0; + image->readFd = -1; image->cacheFd = -1; image->working = (image->cache_map == NULL ); spin_init( &image->lock, PTHREAD_PROCESS_PRIVATE ); @@ -700,6 +697,9 @@ static bool image_load(char *base, char *path, int withUplink) _images[_num_images++] = image; printf( "[DEBUG] Loaded image '%s'\n", image->lower_name ); } + // Keep fd for reading + image->readFd = fdImage; + fdImage = -1; spin_unlock( &_images_lock ); function_return = true; @@ -1191,22 +1191,20 @@ bool image_checkBlocksCrc32(int fd, uint32_t *crc32list, const int *blocks, cons { char buffer[40000]; while ( *blocks != -1 ) { - if ( lseek( fd, (int64_t)*blocks * HASH_BLOCK_SIZE, SEEK_SET ) != (int64_t)*blocks * HASH_BLOCK_SIZE ) { - memlogf( "Seek error" ); - return false; - } uint32_t crc = crc32( 0L, Z_NULL, 0 ); int bytes = 0; - const int bytesToGo = MIN(HASH_BLOCK_SIZE, fileSize - ((int64_t)*blocks * HASH_BLOCK_SIZE)); + const int bytesToGo = MIN( HASH_BLOCK_SIZE, fileSize - ((int64_t)*blocks * HASH_BLOCK_SIZE) ); + off_t readPos = (int64_t)*blocks * HASH_BLOCK_SIZE; while ( bytes < bytesToGo ) { - const int n = MIN((int)sizeof(buffer), bytesToGo - bytes); - const int r = read( fd, buffer, n ); + const int n = MIN( (int)sizeof(buffer), bytesToGo - bytes ); + const int r = pread( fd, buffer, n, readPos ); if ( r <= 0 ) { - memlogf( "Read error" ); + memlogf( "[WARNING] CRC-Check: Read error (errno=%d)", errno ); return false; } crc = crc32( crc, (Bytef*)buffer, r ); bytes += r; + readPos += r; } if ( crc != crc32list[*blocks] ) { printf( "Block %d is %x, should be %x\n", *blocks, crc, crc32list[*blocks] ); diff --git a/src/server/integrity.c b/src/server/integrity.c index 547f8d3..efc54c5 100644 --- a/src/server/integrity.c +++ b/src/server/integrity.c @@ -121,9 +121,9 @@ static void* integrity_main(void * data UNUSED) if ( checkQueue[i].image == NULL ) continue; dnbd3_image_t * const image = image_lock( checkQueue[i].image ); checkQueue[i].image = NULL; + if ( i + 1 == queueLen ) queueLen--; if ( image == NULL ) continue; // We have the image. Call image_release() some time - if ( i + 1 == queueLen ) queueLen--; spin_lock( &image->lock ); if ( image->crc32 != NULL && image->filesize != 0 ) { int const blocks[2] = { checkQueue[i].block, -1 }; @@ -137,15 +137,11 @@ static void* integrity_main(void * data UNUSED) } memcpy( buffer, image->crc32, required ); spin_unlock( &image->lock ); - int fd = open( image->path, O_RDONLY ); - if ( fd >= 0 ) { - if ( image_checkBlocksCrc32( fd, (uint32_t*)buffer, blocks, fileSize ) ) { - //printf( "[DEBUG] CRC check of block %d for %s succeeded :-)\n", blocks[0], image->lower_name ); - } else { - memlogf( "[WARNING] Hash check for block %d of %s failed!", blocks[0], image->lower_name ); - image_updateCachemap( image, blocks[0] * HASH_BLOCK_SIZE, (blocks[0] + 1) * HASH_BLOCK_SIZE, false ); - } - close( fd ); + if ( image_checkBlocksCrc32( image->readFd, (uint32_t*)buffer, blocks, fileSize ) ) { + //printf( "[DEBUG] CRC check of block %d for %s succeeded :-)\n", blocks[0], image->lower_name ); + } else { + memlogf( "[WARNING] Hash check for block %d of %s failed!", blocks[0], image->lower_name ); + image_updateCachemap( image, blocks[0] * HASH_BLOCK_SIZE, (blocks[0] + 1) * HASH_BLOCK_SIZE, false ); } pthread_mutex_lock( &integrityQueueLock ); } else { diff --git a/src/server/net.c b/src/server/net.c index 8e8741a..586b656 100644 --- a/src/server/net.c +++ b/src/server/net.c @@ -126,6 +126,7 @@ void *net_client_handler(void *dnbd3_client) int num; bool bOk = false; + bool hasName = false; serialized_buffer_t payload; char *image_name; @@ -165,19 +166,17 @@ void *net_client_handler(void *dnbd3_client) } else if ( !image->working ) { printf( "[DEBUG] Client requested non-working image '%s' (rid:%d), rejected\n", image_name, (int)rid ); } else { - image_file = open( image->path, O_RDONLY ); - if ( image_file >= 0 ) { - serializer_reset_write( &payload ); - serializer_put_uint16( &payload, PROTOCOL_VERSION ); - serializer_put_string( &payload, image->lower_name ); - serializer_put_uint16( &payload, image->rid ); - serializer_put_uint64( &payload, image->filesize ); - reply.cmd = CMD_SELECT_IMAGE; - reply.size = serializer_get_written_length( &payload ); - if ( send_reply( client->sock, &reply, &payload ) ) { - if ( !client->isServer ) image->atime = time( NULL ); - bOk = true; - } + image_file = image->readFd; + serializer_reset_write( &payload ); + serializer_put_uint16( &payload, PROTOCOL_VERSION ); + serializer_put_string( &payload, image->lower_name ); + serializer_put_uint16( &payload, image->rid ); + serializer_put_uint64( &payload, image->filesize ); + reply.cmd = CMD_SELECT_IMAGE; + reply.size = serializer_get_written_length( &payload ); + if ( send_reply( client->sock, &reply, &payload ) ) { + if ( !client->isServer ) image->atime = time( NULL ); + bOk = true; } } } @@ -192,10 +191,6 @@ void *net_client_handler(void *dnbd3_client) } else if ( !client->isServer && _clientPenalty != 0 ) { usleep( _clientPenalty ); } - if ( host_to_string( &client->host, buffer, sizeof buffer ) ) { - //printf( "[DEBUG] Client %s gets %s\n", buffer, image_name ); - setThreadName( buffer ); - } // client handling mainloop while ( recv_request_header( client->sock, &request ) ) { if ( _shutdown ) break; @@ -290,10 +285,11 @@ void *net_client_handler(void *dnbd3_client) reply.handle = request.handle; fixup_reply( reply ); - pthread_mutex_lock( &client->sendMutex ); + const bool lock = image->uplink != NULL; + if ( lock ) pthread_mutex_lock( &client->sendMutex ); // Send reply header if ( send( client->sock, &reply, sizeof(dnbd3_reply_t), (request.size == 0 ? 0 : MSG_MORE) ) != sizeof(dnbd3_reply_t) ) { - pthread_mutex_unlock( &client->sendMutex ); + if ( lock ) pthread_mutex_unlock( &client->sendMutex ); printf( "[DEBUG] Sending CMD_GET_BLOCK header failed\n" ); goto exit_client_cleanup; } @@ -305,34 +301,42 @@ void *net_client_handler(void *dnbd3_client) while ( done < request.size ) { const ssize_t ret = sendfile( client->sock, image_file, &offset, request.size - done ); if ( ret <= 0 ) { - pthread_mutex_unlock( &client->sendMutex ); + if ( lock ) pthread_mutex_unlock( &client->sendMutex ); printf( "[DEBUG] sendfile failed (image to net. ret=%d, sent %d/%d, errno=%d)\n", (int)ret, (int)done, (int)request.size, (int)errno ); + if ( errno == EBADF || errno == EINVAL || errno == EIO ) image->working = false; goto exit_client_cleanup; } done += ret; } } - pthread_mutex_unlock( &client->sendMutex ); + if ( lock ) pthread_mutex_unlock( &client->sendMutex ); break; case CMD_GET_SERVERS: - client->isServer = false; // Only clients request list of servers // Build list of known working alt servers num = altservers_getMatching( &client->host, server_list, NUMBER_SERVERS ); reply.cmd = CMD_GET_SERVERS; reply.size = num * sizeof(dnbd3_server_entry_t); send_reply( client->sock, &reply, server_list ); + client->isServer = false; // Only clients request list of servers + goto set_name; break; case CMD_KEEPALIVE: reply.cmd = CMD_KEEPALIVE; reply.size = 0; send_reply( client->sock, &reply, NULL ); +set_name: ; + if ( !hasName && host_to_string( &client->host, buffer, sizeof buffer ) ) { + hasName = true; + setThreadName( buffer ); + } break; case CMD_SET_CLIENT_MODE: image->atime = time( NULL ); + client->isServer = false; break; case CMD_GET_CRC32: @@ -355,8 +359,7 @@ void *net_client_handler(void *dnbd3_client) } } } - exit_client_cleanup: ; - if ( image_file != -1 ) close( image_file ); +exit_client_cleanup: ; dnbd3_removeClient( client ); client = dnbd3_freeClient( client ); return NULL ; -- cgit v1.2.3-55-g7522