From 7e0a4560dd7169a3e5177272f99ad5cef51b369a Mon Sep 17 00:00:00 2001 From: Simon Rettberg Date: Wed, 28 Jan 2015 19:06:40 +0100 Subject: [SERVER] Implement proper keep alive for uplinks --- src/server/image.c | 24 ++++++++++++++---------- src/server/uplink.c | 33 ++++++++++++++++----------------- 2 files changed, 30 insertions(+), 27 deletions(-) diff --git a/src/server/image.c b/src/server/image.c index d20a149..81c649d 100644 --- a/src/server/image.c +++ b/src/server/image.c @@ -32,7 +32,7 @@ pthread_spinlock_t _images_lock; static pthread_mutex_t remoteCloneLock = PTHREAD_MUTEX_INITIALIZER; #define NAMELEN 500 -#define CACHELEN 100 +#define CACHELEN 20 typedef struct { char name[NAMELEN]; @@ -890,14 +890,16 @@ dnbd3_image_t* image_getOrClone(char *name, uint16_t revision) const time_t now = time( NULL ); char *cmpname = name; + int useIndex = -1; if ( len >= NAMELEN ) cmpname += 1 + len - NAMELEN; pthread_mutex_lock( &remoteCloneLock ); for (i = 0; i < CACHELEN; ++i) { - if ( remoteCloneCache[i].rid != revision - || remoteCloneCache[i].deadline < now - || strcmp( cmpname, remoteCloneCache[i].name ) != 0 ) continue; - pthread_mutex_unlock( &remoteCloneLock ); // Was recently checked... - return image_get( name, revision, true ); + if ( remoteCloneCache[i].rid == revision && strcmp( cmpname, remoteCloneCache[i].name ) == 0 ) { + useIndex = i; + if ( remoteCloneCache[i].deadline < now ) break; + pthread_mutex_unlock( &remoteCloneLock ); // Was recently checked... + return image_get( name, revision, true ); + } } // Re-check to prevent two clients at the same time triggering this image = image_get( name, revision, true ); @@ -908,10 +910,12 @@ dnbd3_image_t* image_getOrClone(char *name, uint16_t revision) // Reaching this point means we should contact an authority server serialized_buffer_t serialized; // Mark as recently checked - remoteCloneCacheIndex = (remoteCloneCacheIndex + 1) % CACHELEN; - remoteCloneCache[remoteCloneCacheIndex].deadline = now + SERVER_REMOTE_IMAGE_CHECK_CACHETIME; - snprintf( remoteCloneCache[remoteCloneCacheIndex].name, NAMELEN, "%s", cmpname ); - remoteCloneCache[remoteCloneCacheIndex].rid = revision; + if ( useIndex == -1 ) { + useIndex = remoteCloneCacheIndex = (remoteCloneCacheIndex + 1) % CACHELEN; + } + remoteCloneCache[useIndex].deadline = now + SERVER_REMOTE_IMAGE_CHECK_CACHETIME; + snprintf( remoteCloneCache[useIndex].name, NAMELEN, "%s", cmpname ); + remoteCloneCache[useIndex].rid = revision; // Get some alt servers and try to get the image from there dnbd3_host_t servers[4]; int uplinkSock = -1; diff --git a/src/server/uplink.c b/src/server/uplink.c index dea0298..6f91700 100644 --- a/src/server/uplink.c +++ b/src/server/uplink.c @@ -218,7 +218,7 @@ static void* uplink_mainloop(void *data) int numSocks, i, waitTime; int altCheckInterval = SERVER_RTT_DELAY_INIT; int discoverFailCount = 0; - time_t nextAltCheck = 0; + time_t nextAltCheck = 0, nextKeepalive = 0; char buffer[200]; // assert( link != NULL ); @@ -302,7 +302,8 @@ static void* uplink_mainloop(void *data) link->fd = -1; close( events[i].data.fd ); printf( "[DEBUG] Uplink gone away, panic!\n" ); - nextAltCheck = 0; + } else if ( events[i].data.fd == link->signal ) { + printf( "[DEBUG] Error on uplink signal fd!\n" ); } else { printf( "[DEBUG] Error on unknown FD in uplink epoll\n" ); close( events[i].data.fd ); @@ -329,9 +330,18 @@ static void* uplink_mainloop(void *data) } } // Done handling epoll sockets + const time_t now = time( NULL ); + // Send keep alive if nothing is happening + if ( link->fd != -1 && link->replicationHandle == 0 && now > nextKeepalive ) { + nextKeepalive = now + 29; + if ( !uplink_sendKeepalive( link->fd ) ) { + const int fd = link->fd; + link->fd = -1; + close( fd ); + } + } // See if we should trigger an RTT measurement if ( link->rttTestResult == RTT_IDLE || link->rttTestResult == RTT_DONTCHANGE ) { - const time_t now = time( NULL ); if ( now + SERVER_RTT_DELAY_FAILED < nextAltCheck ) { // This probably means the system time was changed - handle this case properly by capping the timeout nextAltCheck = now + SERVER_RTT_DELAY_FAILED / 2; @@ -348,16 +358,6 @@ static void* uplink_mainloop(void *data) } else { // Not complete - do measurement altservers_findUplink( link ); // This will set RTT_INPROGRESS (synchronous) - // Also send a keepalive packet to the currently connected server - if ( link->fd != -1 ) { - if ( !uplink_sendKeepalive( link->fd ) ) { - printf( "[DEBUG] Error sending keep-alive to uplink\n" ); - altservers_serverFailed( &link->currentServer ); - const int fd = link->fd; - link->fd = -1; - close( fd ); - } - } } altCheckInterval = MIN(altCheckInterval + 1, SERVER_RTT_DELAY_MAX); nextAltCheck = now + altCheckInterval; @@ -365,13 +365,12 @@ static void* uplink_mainloop(void *data) } else if ( link->rttTestResult == RTT_NOT_REACHABLE ) { link->rttTestResult = RTT_IDLE; discoverFailCount++; - nextAltCheck = time( NULL ) + (discoverFailCount < 5 ? altCheckInterval : SERVER_RTT_DELAY_FAILED); + nextAltCheck = now + (discoverFailCount < 5 ? altCheckInterval : SERVER_RTT_DELAY_FAILED); } - // TODO: If background replication is disabled, send keepalive every 30 seconds or so #ifdef _DEBUG if ( link->fd != -1 && !link->shutdown ) { bool resend = false; - time_t deadline = time( NULL ) - 15; + const time_t deadline = now - 15; spin_lock( &link->queueLock ); for (i = 0; i < link->queueLen; ++i) { if ( link->queue[i].status != ULR_FREE && link->queue[i].entered < deadline ) { @@ -468,6 +467,7 @@ static void uplink_sendReplicationRequest(dnbd3_connection_t *link) return; } const int len = IMGSIZE_TO_MAPBYTES( image->filesize ) - 1; + // Needs to be 8 (bit->byte, bitmap) const uint32_t requestBlockSize = DNBD3_BLOCK_SIZE * 8; for (int i = 0; i <= len; ++i) { if ( image->cache_map == NULL || link->fd == -1 ) break; @@ -475,7 +475,6 @@ static void uplink_sendReplicationRequest(dnbd3_connection_t *link) link->replicationHandle = 1; // Prevent race condition spin_unlock( &image->lock ); // Unlocked - do not break or continue here... - // Needs to be 8 (bit->byte, bitmap) const uint64_t offset = link->replicationHandle = (uint64_t)i * (uint64_t)requestBlockSize; const uint32_t size = MIN( image->filesize - offset, requestBlockSize ); if ( !dnbd3_get_block( link->fd, offset, size, link->replicationHandle ) ) { -- cgit v1.2.3-55-g7522