diff options
-rw-r--r-- | conf/server.conf | 2 | ||||
-rw-r--r-- | src/server/globals.c | 7 | ||||
-rw-r--r-- | src/server/globals.h | 14 | ||||
-rw-r--r-- | src/server/image.c | 44 | ||||
-rw-r--r-- | src/server/image.h | 2 | ||||
-rw-r--r-- | src/server/uplink.c | 11 |
6 files changed, 69 insertions, 11 deletions
diff --git a/conf/server.conf b/conf/server.conf index f748a4a..332f8a3 100644 --- a/conf/server.conf +++ b/conf/server.conf @@ -13,6 +13,8 @@ isProxy=true backgroundReplication=true ; if isProxy==true and another proxy requests and image that we don't have, should we ask our alt-servers for it? lookupMissingForProxy=true +; create sparse files instead of preallocating; ignored if backgroundReplication=true -- only recommended if cache space is small +sparseFiles=false ; if true (which is the default), images will automatically be removed from the list if they can't be accessed removeMissingImages=true ; timeout in ms for send/recv on connections to uplink servers (used for replication) diff --git a/src/server/globals.c b/src/server/globals.c index b1775f0..0502bbc 100644 --- a/src/server/globals.c +++ b/src/server/globals.c @@ -18,6 +18,7 @@ int _clientPenalty = 0; bool _isProxy = false; bool _backgroundReplication = true; bool _lookupMissingForProxy = true; +bool _sparseFiles = false; bool _removeMissingImages = true; int _uplinkTimeout = SOCKET_TIMEOUT_UPLINK; int _clientTimeout = SOCKET_TIMEOUT_CLIENT; @@ -54,6 +55,7 @@ static int ini_handler(void *custom UNUSED, const char* section, const char* key SAVE_TO_VAR_BOOL( dnbd3, proxyPrivateOnly ); SAVE_TO_VAR_BOOL( dnbd3, backgroundReplication ); SAVE_TO_VAR_BOOL( dnbd3, lookupMissingForProxy ); + SAVE_TO_VAR_BOOL( dnbd3, sparseFiles ); SAVE_TO_VAR_BOOL( dnbd3, removeMissingImages ); SAVE_TO_VAR_BOOL( dnbd3, closeUnusedFd ); SAVE_TO_VAR_UINT( dnbd3, serverPenalty ); @@ -146,6 +148,10 @@ void globals_loadConfig() } } } + if ( _backgroundReplication && _sparseFiles ) { + logadd( LOG_WARNING, "Ignoring 'sparseFiles=true' since backgroundReplication is set to true" ); + _sparseFiles = false; + } // Dump config as interpreted char buffer[2000]; globals_dumpConfig( buffer, sizeof(buffer) ); @@ -257,6 +263,7 @@ size_t globals_dumpConfig(char *buffer, size_t size) PBOOL(isProxy); PBOOL(backgroundReplication); PBOOL(lookupMissingForProxy); + PBOOL(sparseFiles); PBOOL(removeMissingImages); PINT(uplinkTimeout); PINT(clientTimeout); diff --git a/src/server/globals.h b/src/server/globals.h index c1d5d78..d43878f 100644 --- a/src/server/globals.h +++ b/src/server/globals.h @@ -214,6 +214,20 @@ extern bool _backgroundReplication; extern bool _lookupMissingForProxy; /** + * Should we preallocate proxied images right at the start to make + * sure we can cache it entirely, or rather create sparse files + * with holes in them? With sparse files, we just keep writing + * cached blocks to disk until it is full, and only then will we + * start to delete old images. This might be a bit flaky so use + * only in space restricted environments. Also make sure your + * file system actually supports sparse files / files with holes + * in them, or you might get really shitty performance. + * This setting will have no effect if background replication is + * turned on. + */ +extern bool _sparseFiles; + +/** * Port to listen on (default: #define PORT (5003)) */ extern int _listenPort; diff --git a/src/server/image.c b/src/server/image.c index ca00c63..78091f1 100644 --- a/src/server/image.c +++ b/src/server/image.c @@ -49,7 +49,7 @@ static bool image_addToList(dnbd3_image_t *image); static bool image_load(char *base, char *path, int withUplink); static bool image_clone(int sock, char *name, uint16_t revision, uint64_t imageSize); static bool image_calcBlockCrc32(const int fd, const size_t block, const uint64_t realFilesize, uint32_t *crc); -static bool image_ensureDiskSpace(uint64_t size); +static bool image_ensureDiskSpace(uint64_t size, bool force); static uint8_t* image_loadCacheMap(const char * const imagePath, const int64_t fileSize); static uint32_t* image_loadCrcList(const char * const imagePath, const int64_t fileSize, uint32_t *masterCrc); @@ -1125,13 +1125,16 @@ bool image_create(char *image, int revision, uint64_t size) // Try cache map first if ( !file_alloc( fdCache, 0, mapsize ) ) { const int err = errno; - logadd( LOG_ERROR, "Could not allocate %d bytes for %s (errno=%d)", mapsize, cache, err ); - goto failure_cleanup; + logadd( LOG_DEBUG1, "Could not allocate %d bytes for %s (errno=%d)", mapsize, cache, err ); } // Now write image - if ( !file_alloc( fdImage, 0, size ) ) { + if ( !_sparseFiles && !file_alloc( fdImage, 0, size ) ) { const int err = errno; logadd( LOG_ERROR, "Could not allocate %" PRIu64 " bytes for %s (errno=%d)", size, path, err ); + logadd( LOG_ERROR, "It is highly recommended to use a file system that supports preallocating disk" + " space without actually writing all zeroes to the block device." ); + logadd( LOG_ERROR, "If you cannot fix this, try setting sparseFiles=true, but don't expect" + " divine performance during replication." ); goto failure_cleanup; } close( fdImage ); @@ -1275,8 +1278,13 @@ static dnbd3_image_t *loadImageProxy(char * const name, const uint16_t revision, goto server_fail; } pthread_mutex_lock( &reloadLock ); - ok = image_ensureDiskSpace( remoteImageSize ) - && image_clone( sock, name, remoteRid, remoteImageSize ); // This sets up the file+map+crc and loads the img + // Ensure disk space entirely if not using sparse files, otherwise just make sure we have some room at least + if ( _sparseFiles ) { + ok = image_ensureDiskSpace( 2ull * 1024 * 1024 * 1024, false ); // 2GiB, maybe configurable one day + } else { + ok = image_ensureDiskSpace( remoteImageSize + ( 10 * 1024 * 1024 ), false ); // some extra space for cache map etc. + } + ok = ok && image_clone( sock, name, remoteRid, remoteImageSize ); // This sets up the file+map+crc and loads the img pthread_mutex_unlock( &reloadLock ); if ( !ok ) goto server_fail; @@ -1677,14 +1685,30 @@ static bool image_calcBlockCrc32(const int fd, const size_t block, const uint64_ } /** + * Call image_ensureDiskSpace (below), but aquire + * reloadLock first. + */ +bool image_ensureDiskSpaceLocked(uint64_t size, bool force) +{ + bool ret; + pthread_mutex_lock( &reloadLock ); + ret = image_ensureDiskSpace( size, force ); + pthread_mutex_unlock( &reloadLock ); + return ret; +} + +/** * Make sure at least size bytes are available in _basePath. * Will delete old images to make room for new ones. * TODO: Store last access time of images. Currently the - * last access time is reset on server restart. Thus it will - * currently only delete images if server uptime is > 10 hours + * last access time is reset to the file modification time + * on server restart. Thus it will + * currently only delete images if server uptime is > 10 hours. + * This can be overridden by setting force to true, in case + * free space is desperately needed. * Return true iff enough space is available. false in random other cases */ -static bool image_ensureDiskSpace(uint64_t size) +static bool image_ensureDiskSpace(uint64_t size, bool force) { for ( int maxtries = 0; maxtries < 20; ++maxtries ) { uint64_t available; @@ -1694,7 +1718,7 @@ static bool image_ensureDiskSpace(uint64_t size) return true; } if ( available > size ) return true; - if ( dnbd3_serverUptime() < 10 * 3600 ) { + if ( !force && dnbd3_serverUptime() < 10 * 3600 ) { logadd( LOG_INFO, "Only %dMiB free, %dMiB requested, but server uptime < 10 hours...", (int)(available / (1024ll * 1024ll)), (int)(size / (1024 * 1024)) ); return false; diff --git a/src/server/image.h b/src/server/image.h index 213734d..2290744 100644 --- a/src/server/image.h +++ b/src/server/image.h @@ -43,6 +43,8 @@ int image_getCompletenessEstimate(dnbd3_image_t * const image); void image_closeUnusedFd(); +bool image_ensureDiskSpaceLocked(uint64_t size, bool force); + // one byte in the map covers 8 4kib blocks, so 32kib per byte // "+ (1 << 15) - 1" is required to account for the last bit of // the image that is smaller than 32kib diff --git a/src/server/uplink.c b/src/server/uplink.c index aec47cb..b166c6d 100644 --- a/src/server/uplink.c +++ b/src/server/uplink.c @@ -592,10 +592,19 @@ static void uplink_handleReceive(dnbd3_connection_t *link) spin_unlock( &link->image->lock ); // 1) Write to cache file if ( link->image->cacheFd != -1 ) { + bool tryFree = true; uint32_t done = 0; while ( done < inReply.size ) { ret = (int)pwrite( link->image->cacheFd, link->recvBuffer + done, inReply.size - done, start + done ); - if ( ret == -1 && errno == EINTR ) continue; + if ( ret == -1 ) { + if ( errno == EINTR ) continue; + if ( errno == ENOSPC || errno == EDQUOT ) { + // try to free 256MiB + if ( !tryFree || !image_ensureDiskSpaceLocked( 256ull * 1024 * 1024, true ) ) break; + tryFree = false; + continue; // Success, retry write + } + } if ( ret <= 0 ) break; done += (uint32_t)ret; } |