added blockupload dump into status.txt

4 files changed, 123 insertions, 66 deletions

diff --git a/inc/dnbd3/config.h b/inc/dnbd3/config.h
index 53eef64..df9d0e0 100644
--- a/inc/dnbd3/config.h
+++ b/inc/dnbd3/config.h
@@ -50,6 +50,7 @@
 #define COW_MAX_PARALLEL_BACKGROUND_UPLOADS 2 // maximum number of parallel uploads while the image is still mounted
 #define COW_URL_STRING_SIZE 500 // Max string size for an url
 #define COW_SHOW_UL_SPEED 1 // enable display of ul speed in cow status file
+#define COW_MAX_IMAGE_SIZE 1000LL * 1000LL * 1000LL * 1000LL; // Maximum size an image can have(tb*gb*mb*kb)
 // +++++ COW API Endpoints +++++
 #define COW_API_CREATE "%s/api/File/Create"
 #define COW_API_UPDATE "%s/api/File/Update?guid=%s&BlockNumber=%lu"
diff --git a/src/fuse/cowDoc/readme.md b/src/fuse/cowDoc/readme.md
index 858255a..c59d091 100644
--- a/src/fuse/cowDoc/readme.md
+++ b/src/fuse/cowDoc/readme.md
@@ -56,12 +56,12 @@ typedef struct cow_block_metadata
 {
 	atomic_int_least64_t offset;
 	atomic_uint_least64_t timeChanged;
-	atomic_uint_least64_t timeUploaded;
+	atomic_uint_least64_t uploads;
 	atomic_char bitfield[40];
 } cow_block_metadata_t;
 ```
 Each `cow_block_metadata_t` contains a 40 byte so 320 bit bitfield. The bitfield indicates whether the corresponding dnbd3 block contains data or not. For e.g. if the bitfield starts with 01.., the first 4096 contains not data and the next 4096 contain data.
-So each `cow_block_metadata_t` stores the metadata  of up to 320*4096 byte if all bits are set to 1. The offset is the offset where in the data file is the corresponding data stored. The timeChanged property contains the unix when the block was last modified, and the timeUploaded property contains the unix time when the block was last uploaded. It's 0 if the block was never uploaded.
+So each `cow_block_metadata_t` stores the metadata  of up to 320*4096 byte if all bits are set to 1. The offset is the offset where in the data file is the corresponding data stored. The timeChanged property contains the unix when the block was last modified. It's 0 if it was never modified or if the last changes are already uploaded.
 
 
 The L2 arrays and `cow_block_metadata_t` are sorted to original Offsets. So the first L1 pointer, thus the first L2 array, addresses the first 1024 * 320 * 4096 Bytes (L2Size * bitfieldsize * DNBD3Blocksize) of Data and so on.
@@ -107,6 +107,9 @@ Then the corresponding bit in the bitfields will be set and the `timeChanged` wi
 The `workCounter` variable is used here again to make sure that if padding was needed it is done before the fuse request returns.
 
 
+### Block Upload
+For block upload there is an background threads which loop periodically over all cow blocks and checks if `timeChanged` is not 0 and the time difference between now an `timeChanged` is larger than `COW_MIN_UPLOAD_DELAY`. If so, the block will be uploaded. The `timeChanged` before the upload will be temporary stored. After the upload `timeChanged` will be set to 0 if it still has the same time than temporary stored (if not there was an modification while the upload and it needs to be uploaded again). Once the image is unmounted `COW_MIN_UPLOAD_DELAY` is ignored an all blocks if a time of not 0 will be uploaded. The upload is done via an  [rest request](#/api/file/update). There are two different  limits for the number of parallel uploads in the [config.h](#config-variables).
+
 ## Files
 If a new CoW session is started, a new `meta`, `data` and if set so in the Command line arguments a `status.txt` file is created.
 
@@ -137,6 +140,7 @@ it is `done` if the image got dismounted and all blocks are uploaded.
 - `ulspeed` the current upload speed in kb/s.
 
 Once all blocks are uploaded, the state will be set to `done`.
+If you define `COW_DUMP_BLOCK_UPLOADS`, then after the block upload is complete, a list of all blocks and sorted by the number of uploads will be dumped into status.txt.
 
 With the command line parameter `--cowStatStdout` the same output of the stats file will be printed in stdout.
 
@@ -208,13 +212,14 @@ The follwoing configuration variables have been added to ```config.h```.
 #define COW_MAX_PARALLEL_BACKGROUND_UPLOADS 2 // maximum number of parallel uploads while the image is still mounted
 #define COW_URL_STRING_SIZE 500 // Max string size for an url
 #define COW_SHOW_UL_SPEED 1 // enable display of ul speed in cow status file
+#define COW_MAX_IMAGE_SIZE 1000LL * 1000LL * 1000LL * 1000LL; // Maximum size an image can have(tb*gb*mb*kb)
 // +++++ COW API Endpoints +++++
 #define COW_API_CREATE "%s/api/File/Create"
 #define COW_API_UPDATE "%s/api/File/Update?guid=%s&BlockNumber=%lu"
 #define COW_API_START_MERGE "%s/api/File/StartMerge"
 ```
 
-- ```COW_MIN_UPLOAD_DELAY``` defines the minimum time in seconds that must have elapsed since the last modification of a cow block before it is uploaded. This value can be fine tuned. A larger value usually reduces duplicate block uploads. While a lower value usually reduces the time for the final upload after the image got unmounted.
+- ```COW_MIN_UPLOAD_DELAY``` defines the minimum time in seconds that must have elapsed since the last modification of a cow block before it is uploaded. This value can be fine tuned. A larger value usually reduces duplicate block uploads. While a lower value usually reduces the time for the final upload after the image got unmounted. If you define `COW_DUMP_BLOCK_UPLOADS` and have set the command line parameter `--cowStatFile`, then after the block upload is complete, a list of all blocks and sorted by the number of uploads will be dumped into status.txt. This can help adjusting `COW_MIN_UPLOAD_DELAY`.
 
 - ```COW_STATS_UPDATE_TIME``` defines the update frequency in seconds of the stdout print/ stats file update. Setting this to low could impact the performance since it hast to loop over all blocks.
 - ```COW_MAX_PARALLEL_UPLOADS``` defines to maximal number of parallel block uploads. These number is used once the image hast been dismounted and the final blocks are uploaded
diff --git a/src/fuse/cowfile.c b/src/fuse/cowfile.c
index c18b6ec..462c42e 100644
--- a/src/fuse/cowfile.c
+++ b/src/fuse/cowfile.c
@@ -272,7 +272,7 @@ bool mergeRequest()
 		return false;
 	}
 	curl_easy_reset( curl );
-	 curl_mime_free( mime) ;
+	curl_mime_free( mime );
 	return true;
 }
 
@@ -329,26 +329,28 @@ int progress_callback( void *clientp, __attribute__( ( unused ) ) curl_off_t dlT
  * @param speedBuffer ptr to char array that contains the current upload speed.
  */
 
-void updateCowStatsFile( uint64_t inQueue, uint64_t modified, uint64_t idle, char * speedBuffer )
+void updateCowStatsFile( uint64_t inQueue, uint64_t modified, uint64_t idle, char *speedBuffer )
 {
 	char buffer[300];
 	char state[30];
-	if( uploadLoop ) {
+	if ( uploadLoop ) {
 		snprintf( state, 30, "%s", "backgroundUpload" );
-	} else if( !uploadLoopDone ) {
+	} else if ( !uploadLoopDone ) {
 		snprintf( state, 30, "%s", "uploading" );
 	} else {
 		snprintf( state, 30, "%s", "done" );
 	}
 
-	int len = snprintf( buffer, 300, "state=%s\n"
-									 "inQueue=%"PRIu64"\n"
-									 "modifiedBlocks=%"PRIu64"\n"
-									 "idleBlocks=%"PRIu64"\n"
-									 "totalBlocksUploaded=%"PRIu64"\n"
-									 "activeUploads:%i\n"
-									 "%s=%s",
-			state,  inQueue, modified, idle, totalBlocksUploaded, activeUploads,COW_SHOW_UL_SPEED ? "ulspeed" : "", speedBuffer );
+	int len = snprintf( buffer, 300,
+			"state=%s\n"
+			"inQueue=%" PRIu64 "\n"
+			"modifiedBlocks=%" PRIu64 "\n"
+			"idleBlocks=%" PRIu64 "\n"
+			"totalBlocksUploaded=%" PRIu64 "\n"
+			"activeUploads:%i\n"
+			"%s=%s",
+			state, inQueue, modified, idle, totalBlocksUploaded, activeUploads, COW_SHOW_UL_SPEED ? "ulspeed" : "",
+			speedBuffer );
 
 	if ( statStdout ) {
 		logadd( LOG_INFO, "%s", buffer );
@@ -361,8 +363,46 @@ void updateCowStatsFile( uint64_t inQueue, uint64_t modified, uint64_t idle, cha
 		if ( ftruncate( cow.fhs, 43 + len ) ) {
 			logadd( LOG_WARNING, "Could not truncate cow status file" );
 		}
+#ifdef COW_DUMP_BLOCK_UPLOADS
+		if ( !uploadLoop && uploadLoopDone ) {
+			dumpBlockUploads();
+		}
+#endif
 	}
+}
+int cmpfunc( const void *a, const void *b )
+{
+	return (int)( ( (cow_block_upload_statistics_t *)b )->uploads - ( (cow_block_upload_statistics_t *)a )->uploads );
+}
+/**
+ * @brief writes all block numbers sorted by the number of uploads into the statsfile
+ * 
+ */
+void dumpBlockUploads()
+{
+	long unsigned int l1MaxOffset = 1 + ( ( metadata->imageSize - 1 ) / COW_L2_STORAGE_CAPACITY );
+
+	cow_block_upload_statistics_t blockUploads[l1MaxOffset * COW_L2_SIZE];
+	uint64_t currentBlock = 0;
+	for ( long unsigned int l1Offset = 0; l1Offset < l1MaxOffset; l1Offset++ ) {
+		if ( cow.l1[l1Offset] == -1 ) {
+			continue;
+		}
+		for ( int l2Offset = 0; l2Offset < COW_L2_SIZE; l2Offset++ ) {
+			cow_block_metadata_t *block = ( cow.firstL2[cow.l1[l1Offset]] + l2Offset );
 
+			blockUploads[currentBlock].uploads = block->uploads;
+			blockUploads[currentBlock].blocknumber = ( l1Offset * COW_L2_SIZE + l2Offset );
+			currentBlock++;
+		}
+	}
+	qsort( blockUploads, currentBlock, sizeof( cow_block_upload_statistics_t ), cmpfunc );
+	lseek( cow.fhs, 0, SEEK_END );
+
+	dprintf( cow.fhs, "\n\nblocknumber: uploads\n==Block Upload Dump===\n" );
+	for ( uint64_t i = 0; i < currentBlock; i++ ) {
+		dprintf( cow.fhs, "%" PRIu64 ": %" PRIu64 " \n", blockUploads[i].blocknumber, blockUploads[i].uploads );
+	}
 }
 
 /**
@@ -371,7 +411,7 @@ void updateCowStatsFile( uint64_t inQueue, uint64_t modified, uint64_t idle, cha
  * @param cm Curl_multi
  * @param curlUploadBlock cow_curl_read_upload_t containing the data for the block to upload
  */
-bool addUpload( CURLM *cm, cow_curl_read_upload_t *curlUploadBlock, struct curl_slist * headers)
+bool addUpload( CURLM *cm, cow_curl_read_upload_t *curlUploadBlock, struct curl_slist *headers )
 {
 	CURL *eh = curl_easy_init();
 
@@ -392,7 +432,7 @@ bool addUpload( CURLM *cm, cow_curl_read_upload_t *curlUploadBlock, struct curl_
 		curl_easy_setopt( eh, CURLOPT_XFERINFOFUNCTION, progress_callback );
 		curl_easy_setopt( eh, CURLOPT_XFERINFODATA, eh );
 	}
-	curl_easy_setopt( eh, CURLOPT_HTTPHEADER, headers);
+	curl_easy_setopt( eh, CURLOPT_HTTPHEADER, headers );
 	curl_multi_add_handle( cm, eh );
 
 	return true;
@@ -408,18 +448,18 @@ bool addUpload( CURLM *cm, cow_curl_read_upload_t *curlUploadBlock, struct curl_
  * @return true returned if upload was successful or retries still possible
  * @return false returned if upload was unsuccessful
  */
-bool finishUpload( CURLM *cm, CURLMsg *msg, struct curl_slist * headers )
+bool finishUpload( CURLM *cm, CURLMsg *msg, struct curl_slist *headers )
 {
 	bool status = true;
 	cow_curl_read_upload_t *curlUploadBlock;
 	CURLcode res;
 	CURLcode res2;
 	res = curl_easy_getinfo( msg->easy_handle, CURLINFO_PRIVATE, &curlUploadBlock );
-	
+
 	long http_code = 0;
 	res2 = curl_easy_getinfo( msg->easy_handle, CURLINFO_RESPONSE_CODE, &http_code );
 
-	if ( res != CURLE_OK || res2 != CURLE_OK || http_code != 200 ||msg->msg != CURLMSG_DONE ) {
+	if ( res != CURLE_OK || res2 != CURLE_OK || http_code != 200 || msg->msg != CURLMSG_DONE ) {
 		curlUploadBlock->fails++;
 		logadd( LOG_ERROR, "COW_API_UPDATE  failed %i/5: %s\n", curlUploadBlock->fails,
 				curl_easy_strerror( msg->data.result ) );
@@ -456,7 +496,8 @@ CLEANUP:
  * @return true returned if all upload's were successful 
  * @return false returned if  one ore more upload's failed.
  */
-bool MessageHandler( CURLM *cm, atomic_int *activeUploads, bool breakIfNotMax, bool foregroundUpload, struct curl_slist * headers )
+bool MessageHandler(
+		CURLM *cm, atomic_int *activeUploads, bool breakIfNotMax, bool foregroundUpload, struct curl_slist *headers )
 {
 	CURLMsg *msg;
 	int msgsLeft = -1;
@@ -469,8 +510,9 @@ bool MessageHandler( CURLM *cm, atomic_int *activeUploads, bool breakIfNotMax, b
 				status = false;
 			}
 		}
-		if ( breakIfNotMax && *activeUploads < ( foregroundUpload ? COW_MAX_PARALLEL_UPLOADS
-																		: COW_MAX_PARALLEL_BACKGROUND_UPLOADS ) ) {
+		if ( breakIfNotMax
+				&& *activeUploads
+						< ( foregroundUpload ? COW_MAX_PARALLEL_UPLOADS : COW_MAX_PARALLEL_BACKGROUND_UPLOADS ) ) {
 			break;
 		}
 		// ony wait if there are active uploads
@@ -494,7 +536,7 @@ bool MessageHandler( CURLM *cm, atomic_int *activeUploads, bool breakIfNotMax, b
 bool uploaderLoop( bool ignoreMinUploadDelay, CURLM *cm )
 {
 	bool success = true;
-	struct curl_slist * headers = NULL;
+	struct curl_slist *headers = NULL;
 	headers = curl_slist_append( headers, "Content-Type: application/octet-stream" );
 
 	long unsigned int l1MaxOffset = 1 + ( ( metadata->imageSize - 1 ) / COW_L2_STORAGE_CAPACITY );
@@ -513,8 +555,9 @@ bool uploaderLoop( bool ignoreMinUploadDelay, CURLM *cm )
 						if ( !MessageHandler( cm, &activeUploads, true, ignoreMinUploadDelay, headers ) ) {
 							success = false;
 						}
-					} while ( !( activeUploads < ( ignoreMinUploadDelay ? COW_MAX_PARALLEL_UPLOADS : COW_MAX_PARALLEL_BACKGROUND_UPLOADS ) )
-														&& activeUploads );
+					} while ( !( activeUploads < ( ignoreMinUploadDelay ? COW_MAX_PARALLEL_UPLOADS
+																						 : COW_MAX_PARALLEL_BACKGROUND_UPLOADS ) )
+							&& activeUploads );
 					cow_curl_read_upload_t *b = malloc( sizeof( cow_curl_read_upload_t ) );
 					b->block = block;
 					b->blocknumber = ( l1Offset * COW_L2_SIZE + l2Offset );
@@ -522,7 +565,7 @@ bool uploaderLoop( bool ignoreMinUploadDelay, CURLM *cm )
 					b->position = 0;
 					b->time = block->timeChanged;
 					addUpload( cm, b, headers );
-					if( !ignoreMinUploadDelay && !uploadLoop ) {
+					if ( !ignoreMinUploadDelay && !uploadLoop ) {
 						goto DONE;
 					}
 				}
@@ -533,27 +576,27 @@ DONE:
 	while ( activeUploads > 0 ) {
 		MessageHandler( cm, &activeUploads, false, ignoreMinUploadDelay, headers );
 	}
-	curl_slist_free_all(headers);
+	curl_slist_free_all( headers );
 	return success;
 }
 
 
-
 /**
  * @brief Computes the data for the status to the stdout/statfile every COW_STATS_UPDATE_TIME seconds.
  * 
  */
 
-void * cowfile_statUpdater(__attribute__( ( unused ) ) void *something ) {
-	uint64_t lastUpdateTime = time(NULL);
+void *cowfile_statUpdater( __attribute__( ( unused ) ) void *something )
+{
+	uint64_t lastUpdateTime = time( NULL );
 
-	while( !uploadLoopDone ) {
-		sleep(COW_STATS_UPDATE_TIME);
+	while ( !uploadLoopDone ) {
+		sleep( COW_STATS_UPDATE_TIME );
 		int modified = 0;
 		int inQueue = 0;
 		int idle = 0;
 		long unsigned int l1MaxOffset = 1 + ( ( metadata->imageSize - 1 ) / COW_L2_STORAGE_CAPACITY );
-		uint64_t now = time(NULL);
+		uint64_t now = time( NULL );
 		for ( long unsigned int l1Offset = 0; l1Offset < l1MaxOffset; l1Offset++ ) {
 			if ( cow.l1[l1Offset] == -1 ) {
 				continue;
@@ -563,10 +606,10 @@ void * cowfile_statUpdater(__attribute__( ( unused ) ) void *something ) {
 				if ( block->offset == -1 ) {
 					continue;
 				}
-				if ( block->timeChanged != 0) {
-					if( !uploadLoop || now >  block->timeChanged + COW_MIN_UPLOAD_DELAY ) {
+				if ( block->timeChanged != 0 ) {
+					if ( !uploadLoop || now > block->timeChanged + COW_MIN_UPLOAD_DELAY ) {
 						inQueue++;
-					} else { 
+					} else {
 						modified++;
 					}
 				} else {
@@ -577,17 +620,15 @@ void * cowfile_statUpdater(__attribute__( ( unused ) ) void *something ) {
 		char speedBuffer[20];
 
 		if ( COW_SHOW_UL_SPEED ) {
-			now = time(NULL);
+			now = time( NULL );
 			uint64_t bytes = atomic_exchange( &bytesUploaded, 0 );
-			snprintf( speedBuffer, 20, "%.2f",
-				(double)( ( bytes  ) / ( 1 + now -  lastUpdateTime  ) / 1000 ) );
-			
+			snprintf( speedBuffer, 20, "%.2f", (double)( ( bytes ) / ( 1 + now - lastUpdateTime ) / 1000 ) );
+
 			lastUpdateTime = now;
 		}
 
-		
-		updateCowStatsFile( inQueue, modified, idle,  speedBuffer);
-		 
+
+		updateCowStatsFile( inQueue, modified, idle, speedBuffer );
 	}
 }
 
@@ -666,7 +707,7 @@ bool createCowStatsFile( char *path )
  * @param imageSizePtr 
  */
 bool cowfile_init( char *path, const char *image_Name, uint16_t imageVersion, atomic_uint_fast64_t **imageSizePtr,
-		char *serverAddress, bool sStdout, bool sfile)
+		char *serverAddress, bool sStdout, bool sfile )
 {
 	statStdout = sStdout;
 	statFile = sfile;
@@ -688,11 +729,11 @@ bool cowfile_init( char *path, const char *image_Name, uint16_t imageVersion, at
 
 	int maxPageSize = 8192;
 
-	// TODO IMAGE NAME IS FIXED
+
 	size_t metaDataSizeHeader = sizeof( cowfile_metadata_header_t ) + strlen( image_Name );
 
 
-	cow.maxImageSize = 1000LL * 1000LL * 1000LL * 1000LL; // tb*gb*mb*kb todo make this changeable
+	cow.maxImageSize = COW_MAX_IMAGE_SIZE;
 	cow.l1Size = ( ( cow.maxImageSize + COW_L2_STORAGE_CAPACITY - 1LL ) / COW_L2_STORAGE_CAPACITY );
 
 	// size of l1 array + number of l2's * size of l2
@@ -763,7 +804,7 @@ bool cowfile_init( char *path, const char *image_Name, uint16_t imageVersion, at
 
 	createCowStatsFile( path );
 	pthread_create( &tidCowUploader, NULL, &cowfile_uploader, NULL );
-	if ( statFile || statStdout) { 
+	if ( statFile || statStdout ) {
 		pthread_create( &tidStatUpdater, NULL, &cowfile_statUpdater, NULL );
 	}
 	return true;
@@ -775,7 +816,7 @@ bool cowfile_init( char *path, const char *image_Name, uint16_t imageVersion, at
  * @param path where the meta & data file is located 
  * @param imageSizePtr 
  */
-bool cowfile_load( char *path, atomic_uint_fast64_t **imageSizePtr, char *serverAddress,  bool sStdout, bool sFile )
+bool cowfile_load( char *path, atomic_uint_fast64_t **imageSizePtr, char *serverAddress, bool sStdout, bool sFile )
 {
 	statStdout = sStdout;
 	statFile = sFile;
@@ -874,7 +915,7 @@ bool cowfile_load( char *path, atomic_uint_fast64_t **imageSizePtr, char *server
 	createCowStatsFile( path );
 	pthread_create( &tidCowUploader, NULL, &cowfile_uploader, NULL );
 
-	if ( statFile || statStdout) { 
+	if ( statFile || statStdout ) {
 		pthread_create( &tidStatUpdater, NULL, &cowfile_statUpdater, NULL );
 	}
 
@@ -953,7 +994,7 @@ static bool createL2Block( int l1Offset )
 	if ( cow.l1[l1Offset] == -1 ) {
 		for ( int i = 0; i < COW_L2_SIZE; i++ ) {
 			cow.firstL2[metadata->nextL2][i].offset = -1;
-			cow.firstL2[metadata->nextL2][i].timeChanged =  ATOMIC_VAR_INIT( 0 );
+			cow.firstL2[metadata->nextL2][i].timeChanged = ATOMIC_VAR_INIT( 0 );
 			cow.firstL2[metadata->nextL2][i].uploads = ATOMIC_VAR_INIT( 0 );
 			for ( int j = 0; j < COW_BITFIELD_SIZE; j++ ) {
 				cow.firstL2[metadata->nextL2][i].bitfield[j] = ATOMIC_VAR_INIT( 0 );
@@ -983,7 +1024,6 @@ static void finishWriteRequest( fuse_req_t req, cow_request_t *cowRequest )
 	} else {
 		metadata->imageSize = MAX( metadata->imageSize, cowRequest->bytesWorkedOn + cowRequest->fuseRequestOffset );
 		if ( cowRequest->replyAttr ) {
-			//TODO HANDLE ERROR
 			image_ll_getattr( req, cowRequest->ino, cowRequest->fi );
 
 		} else {
@@ -1007,8 +1047,8 @@ static void writePaddedBlock( cow_sub_request_t *sRequest )
 	//copy write Data
 	memcpy( ( sRequest->writeBuffer + ( sRequest->inBlockOffset % DNBD3_BLOCK_SIZE ) ), sRequest->writeSrc,
 			sRequest->size );
-	writeData( sRequest->writeBuffer, DNBD3_BLOCK_SIZE, (ssize_t)sRequest->size, sRequest->cowRequest,
-			sRequest->block, ( sRequest->inBlockOffset - ( sRequest->inBlockOffset % DNBD3_BLOCK_SIZE ) ) );
+	writeData( sRequest->writeBuffer, DNBD3_BLOCK_SIZE, (ssize_t)sRequest->size, sRequest->cowRequest, sRequest->block,
+			( sRequest->inBlockOffset - ( sRequest->inBlockOffset % DNBD3_BLOCK_SIZE ) ) );
 
 
 	if ( atomic_fetch_sub( &sRequest->cowRequest->workCounter, 1 ) == 1 ) {
@@ -1034,7 +1074,7 @@ static void padBlockFromRemote( fuse_req_t req, off_t offset, cow_request_t *cow
 		writeData( buf, DNBD3_BLOCK_SIZE, (ssize_t)size, cowRequest, block, inBlockOffset );
 		return;
 	}
-	cow_sub_request_t *sRequest = malloc( sizeof( cow_sub_request_t ) + DNBD3_BLOCK_SIZE);
+	cow_sub_request_t *sRequest = malloc( sizeof( cow_sub_request_t ) + DNBD3_BLOCK_SIZE );
 	sRequest->callback = writePaddedBlock;
 	sRequest->inBlockOffset = inBlockOffset;
 	sRequest->block = block;
@@ -1055,10 +1095,11 @@ static void padBlockFromRemote( fuse_req_t req, off_t offset, cow_request_t *cow
 
 	atomic_fetch_add( &cowRequest->workCounter, 1 );
 	if ( !connection_read( &sRequest->dRequest ) ) {
-		atomic_fetch_sub( &cowRequest->workCounter, 1 );
-		// todo check if not  now
 		cowRequest->errorCode = EIO;
 		free( sRequest );
+		if ( atomic_fetch_sub( &sRequest->cowRequest->workCounter, 1 ) == 1 ) {
+			finishWriteRequest( sRequest->dRequest.fuse_req, sRequest->cowRequest );
+		}
 		return;
 	}
 }
@@ -1115,7 +1156,6 @@ void cowfile_write( fuse_req_t req, cow_request_t *cowRequest, off_t offset, siz
 		// half end block will be padded with original write
 		pSize = pSize - ( ( pSize + offset ) % DNBD3_BLOCK_SIZE );
 		atomic_fetch_add( &cowRequest->workCounter, 1 );
-		//TODO FIX that its actually 0
 		cowfile_write( req, cowRequest, metadata->imageSize, pSize );
 	}
 
@@ -1199,9 +1239,9 @@ void cowfile_write( fuse_req_t req, cow_request_t *cowRequest, off_t offset, siz
  * @param buffer into which the data is to be written
  * @param workCounter workCounter is increased by one and later reduced by one again when the request is completed.
  */
-static void readRemote( fuse_req_t req, off_t offset, ssize_t size, char * buffer, cow_request_t *cowRequest )
+static void readRemote( fuse_req_t req, off_t offset, ssize_t size, char *buffer, cow_request_t *cowRequest )
 {
-	cow_sub_request_t *sRequest = malloc( sizeof( cow_sub_request_t ));
+	cow_sub_request_t *sRequest = malloc( sizeof( cow_sub_request_t ) );
 	sRequest->callback = readRemoteData;
 	sRequest->dRequest.length = (uint32_t)size;
 	sRequest->dRequest.offset = offset;
@@ -1211,10 +1251,13 @@ static void readRemote( fuse_req_t req, off_t offset, ssize_t size, char * buffe
 
 	atomic_fetch_add( &cowRequest->workCounter, 1 );
 	if ( !connection_read( &sRequest->dRequest ) ) {
-		atomic_fetch_sub( &cowRequest->workCounter, 1 );
-		//TODO ChECK IF NOT  0  Now
 		cowRequest->errorCode = EIO;
 		free( sRequest );
+		if ( atomic_fetch_sub( &cowRequest->workCounter, 1 ) == 1 ) {
+			fuse_reply_buf( req, cowRequest->readBuffer, cowRequest->bytesWorkedOn );
+		}
+		free( cowRequest->readBuffer );
+		free( cowRequest );
 		return;
 	}
 }
@@ -1282,7 +1325,8 @@ void cowfile_read( fuse_req_t req, size_t size, off_t offset )
 		if ( doRead || searchOffset >= endOffset ) {
 			ssize_t sizeToRead = MIN( searchOffset, endOffset ) - lastReadOffset;
 			if ( !isLocal ) {
-				readRemote( req, lastReadOffset, sizeToRead, cowRequest->readBuffer + ( lastReadOffset - offset ), cowRequest );
+				readRemote(
+						req, lastReadOffset, sizeToRead, cowRequest->readBuffer + ( lastReadOffset - offset ), cowRequest );
 			} else {
 				// Compute the offset in the data file where the read starts
 				off_t localRead =
@@ -1339,11 +1383,11 @@ fail:;
 void cowfile_close()
 {
 	uploadLoop = false;
-	if ( statFile || statStdout) { 
+	if ( statFile || statStdout ) {
 		pthread_join( tidStatUpdater, NULL );
 	}
 	pthread_join( tidCowUploader, NULL );
-	
+
 	if ( curl ) {
 		curl_global_cleanup();
 		curl_easy_cleanup( curl );
diff --git a/src/fuse/cowfile.h b/src/fuse/cowfile.h
index 9e4f1fc..05beb0e 100644
--- a/src/fuse/cowfile.h
+++ b/src/fuse/cowfile.h
@@ -79,7 +79,7 @@ typedef struct cow_sub_request
 	size_t size;
 	off_t inBlockOffset;
 	const char *writeSrc;
-	char * buffer;
+	char *buffer;
 	cow_block_metadata_t *block;
 	cow_callback callback;
 	cow_request_t *cowRequest;
@@ -98,6 +98,13 @@ typedef struct cow_curl_read_upload
 } cow_curl_read_upload_t;
 
 
+typedef struct cow_block_upload_statistics
+{
+	uint64_t blocknumber;
+	uint64_t uploads;
+} cow_block_upload_statistics_t;
+
+
 typedef int32_t l1;
 typedef cow_block_metadata_t l2[COW_L2_SIZE];