From 46d390f9a1cdd47d1f03557e73a32c12122135cb Mon Sep 17 00:00:00 2001 From: Michael Scherle Date: Sun, 14 Oct 2018 16:06:37 -0700 Subject: Fixes and also included the Image merging programm --- src/fuse/cow.c | 323 +++++++++++++++++++++++++++++++-------------------------- 1 file changed, 177 insertions(+), 146 deletions(-) (limited to 'src/fuse/cow.c') diff --git a/src/fuse/cow.c b/src/fuse/cow.c index 1834ce7..37345a1 100644 --- a/src/fuse/cow.c +++ b/src/fuse/cow.c @@ -119,6 +119,11 @@ static cow_request* removeCowRequest(cow_requests_queue queue,cow_request *reque } +void writeImageSizeToFile(uint64_t size){ + lseek(fh,sizeof(unsigned int),SEEK_SET); + write(fh, &size, sizeof(uint64_t)); +} + bool create_cow_file(char *cow_path, char *image_Name,uint64_t imageSize){ @@ -129,10 +134,11 @@ bool create_cow_file(char *cow_path, char *image_Name,uint64_t imageSize){ } write(fh,&version,sizeof(unsigned int)); + write(fh,&imageSize,sizeof(uint64_t)); int nameLenght =(unsigned int) strlen(image_Name); write(fh,&nameLenght,sizeof(unsigned int)); write(fh,image_Name,sizeof(char)*strlen(image_Name)); - write(fh,&imageSize,sizeof(uint64_t)); + int pageSize = getpagesize(); write(fh,&pageSize,sizeof(int)); imageBlockCount = (MaxImageSizeInBytes/(1024*1024)); @@ -189,12 +195,60 @@ bool create_cow_file(char *cow_path, char *image_Name,uint64_t imageSize){ +bool load_cow_file(char *cow_path, uint64_t imageSize){ + remoteImageSize =(size_t) imageSize; + if((fh = open (cow_path, O_RDONLY))==-1){ + printf( "Could not open Cow File.\n" ); + return false; + } + unsigned int version ; + uint64_t size; + read(fh,&version,sizeof(unsigned int)); + read(fh,&size,sizeof(uint64_t)); + int l; + read(fh,&l,sizeof(int)); + + char *buffer = malloc(sizeof(char)*(l+1)); + read(fh,buffer,l*sizeof(char)); + buffer[l]='\0'; + + int pageSize; + read(fh,&pageSize,sizeof(int)); + if(debug){ + printf("Version: %u\n",version); + printf("länge: %i \n",l); + printf("Image Name: %s\n",buffer); + printf("Size: %ld\n", (long)size); + printf("pageSize: %i\n", (pageSize)); + } + free(buffer); + //TODO Image Validation + + + setImagesize(size); + off_t mmapStart = lseek(fh, 0L, SEEK_CUR); + int maxPageSize = 8192; + mmapStart= ((mmapStart+maxPageSize-1)/maxPageSize)*maxPageSize; + imageBlockCount = (MaxImageSizeInBytes/(1024*1024)); + filePointers = mmap(NULL,imageBlockCount*sizeof(off_t), PROT_READ, MAP_SHARED,fh, mmapStart); + if(filePointers == MAP_FAILED ){ + printf("Error creating mmap in COW File.\n%s\nBye.\n ", strerror(errno)); + close(fh); + return false; + } + signalInit(); + if(pthread_spin_init( &requestsQueueLock, PTHREAD_PROCESS_PRIVATE ) != 0){ + printf("Spinlock init failure"); + } + return true; +} + + bool createBigBlock(unsigned long id){ - //goto File end and then to next pagesize + //goto File end and then to next 4096 Block off_t blockStart = lseek(fh,0, SEEK_END); - int pageSize = getpagesize(); - blockStart= ((blockStart+pageSize-1)/pageSize)*pageSize; + blockStart= ((blockStart+4096-1)/4096)*4096; filePointers[id] = blockStart; // go to next Page size off_t currentFilePos= lseek(fh,blockStart, SEEK_SET); @@ -202,7 +256,7 @@ bool createBigBlock(unsigned long id){ write(fh,&blockState,sizeof(int)*8); // go to next Page size - currentFilePos = (((currentFilePos+sizeof(int)*8)+pageSize-1)/pageSize)*pageSize; + currentFilePos = (((currentFilePos+sizeof(int)*8)+4096-1)/4096)*4096; currentFilePos = lseek(fh,currentFilePos, SEEK_SET); char data[256*4096] = {0}; write(fh,&data,sizeof(char)*4096*256); @@ -252,10 +306,7 @@ bool queckQueuesForDependencies(cow_request *request,cow_requests_queue queue){ return foundDependencie; } - - cow_request getAccess(off_t offset,size_t size){ - // TODO LOCK THIS ON BLOCKSIZE cow_request request; request.offset = (offset-(offset % 4096)); request.end =(offset+(off_t)size)+( (offset+(off_t)size)% 4096)-1; @@ -267,131 +318,54 @@ cow_request getAccess(off_t offset,size_t size){ request.myDependencies[i] = NULL; } pthread_spin_lock( &requestsQueueLock ); - if(queckQueuesForDependencies(&request, cowRequestsactive)){ queckQueuesForDependencies(&request, cowRequestsQueued); enqueueCowRequest(cowRequestsQueued,&request); pthread_spin_unlock( &requestsQueueLock ); - int ret = signal_wait( request.signal, 200000 ); - if(ret<0){ - //TODO - printf("Error CowRequest timed out"); + int ret = -1; + while(ret<0){ + int ret = signal_wait( request.signal, 5000 ); + if(ret<0){ + printf("Error Cow Request timed out"); + } } signalPut(request.signal); - }else{ enqueueCowRequest(cowRequestsactive,&request); pthread_spin_unlock( &requestsQueueLock ); } return request; - // CHECK AKTIVE -> not enque start - - - - /* - pthread_mutex_lock(&bigBlockLockArray_mutex); - if( (!TestBit(bigBlockLockArray,id1)) &&(!TestBit(bigBlockLockArray,id2))){ - SetBit(bigBlockLockArray,id1); - SetBit(bigBlockLockArray,id2); - pthread_mutex_unlock(&bigBlockLockArray_mutex); - }else{ - cow_request request; - request.bigBlockStartId = id1; - request.bigBlockEndId = id2; - request.signal = signalGet(); - enqueueCowRequest(&request); - pthread_mutex_unlock(&bigBlockLockArray_mutex); - - int ret = signal_wait( request.signal, 200000 ); - if(ret<0){ - //TODO - printf("Error CowRequest timed out"); - } - signalPut(request.signal); - } - */ } void closeAcccess(cow_request *request){ - pthread_spin_lock( &requestsQueueLock ); - removeCowRequest(cowRequestsactive,request); - for (int i = 0;i< 6; i++){ + pthread_spin_lock(&requestsQueueLock); + removeCowRequest(cowRequestsactive, request); + for (int i = 0; i< 6; i++){ cow_request *otherRequest = request->dependencies[i]; if(otherRequest != NULL){ - bool canStart=true; - for (int j = 0;j< 6; j++){ - if(otherRequest->myDependencies[j]==otherRequest){ - otherRequest->myDependencies[j]=NULL; - }else if(otherRequest->myDependencies[j]!=NULL){ + bool canStart = true; + for (int j = 0; j< 6; j++){ + if(otherRequest->myDependencies[j] == request){ + otherRequest->myDependencies[j] = NULL; + }else if(otherRequest->myDependencies[j] != NULL){ canStart=false; } - } if(canStart){ - - removeCowRequest(cowRequestsQueued,otherRequest); - enqueueCowRequest(cowRequestsactive,otherRequest); + removeCowRequest(cowRequestsQueued, otherRequest); + enqueueCowRequest(cowRequestsactive, otherRequest); signal_call(otherRequest->signal); } } request->dependencies[i] = NULL; } - pthread_spin_unlock( &requestsQueueLock ); - - - /* - pthread_mutex_lock(&bigBlockLockArray_mutex); - pthread_spin_lock( &cow_requests.lock ); - ClearBit(bigBlockLockArray,id1); - ClearBit(bigBlockLockArray,id2); - if(cow_requests.head !=NULL){ - cow_request *iterator,*prev = NULL; - for ( iterator = cow_requests.head; iterator != NULL; iterator = iterator->next ) { - if((iterator->bigBlockStartId == id1 )||(iterator->bigBlockEndId == id1 )){ - - if((!TestBit(bigBlockLockArray,iterator->bigBlockStartId))&&(!TestBit(bigBlockLockArray,iterator->bigBlockEndId))){ - SetBit(bigBlockLockArray,iterator->bigBlockStartId); - SetBit(bigBlockLockArray,iterator->bigBlockEndId); - removeFromQueueUnsafe(iterator, prev); - signal_call(iterator->signal); - - if(id1 == id2){ - pthread_spin_unlock( &cow_requests.lock); - pthread_mutex_unlock(&bigBlockLockArray_mutex); - return; - }else{ - id1 = id2; - } - } - }else if((iterator->bigBlockStartId == id2 )||(iterator->bigBlockEndId == id2 )){ - if((!TestBit(bigBlockLockArray,iterator->bigBlockStartId))&&(!TestBit(bigBlockLockArray,iterator->bigBlockEndId))){ - removeFromQueueUnsafe(iterator, prev); - SetBit(bigBlockLockArray,iterator->bigBlockStartId); - SetBit(bigBlockLockArray,iterator->bigBlockEndId); - signal_call(iterator->signal); - - if(id1 == id2){ - pthread_spin_unlock( &cow_requests.lock ); - pthread_mutex_unlock(&bigBlockLockArray_mutex); - return; - }else{ - id2 = id1; - } - } - } - prev = iterator; - //iterator++; - } - } - pthread_spin_unlock( &cow_requests.lock ); - pthread_mutex_unlock(&bigBlockLockArray_mutex); - */ + pthread_spin_unlock(&requestsQueueLock); } int write_cow(const char *data, size_t size, off_t offset) { - size_t totalWrittenBytes = 0; + int writtenBytes = 0; size_t sizeToBigBlock = 0; off_t bigBlockOffset = 0; @@ -399,7 +373,7 @@ int write_cow(const char *data, size_t size, off_t offset) { unsigned long bigBlockStartId = (offset)/(4096*256); unsigned long bigBlockId = bigBlockStartId; cow_request request = getAccess(offset,size); - while(totalWrittenBytes < size){ + while(writtenBytes < size){ @@ -408,17 +382,32 @@ int write_cow(const char *data, size_t size, off_t offset) { // how much i can write in this block - sizeToBigBlock = 4096*256 - (bigBlockOffset -bigBlockStart); + //TODO CHECK THIS + //sizeToBigBlock = 4096*256 - (bigBlockOffset -bigBlockStart); + sizeToBigBlock = 4096*256 - bigBlockOffset; if((size-writtenBytes)< sizeToBigBlock){ sizeToBigBlock = size-writtenBytes; } - writtenBytes = writeToBigBlock(bigBlockId,data+totalWrittenBytes,bigBlockOffset,sizeToBigBlock); - totalWrittenBytes +=writtenBytes; + writtenBytes += writeToBigBlock(bigBlockId,data+writtenBytes,bigBlockOffset,sizeToBigBlock); + bigBlockId++; } + /////////////////////////////// + char *tmp = malloc(size); + cow_read(tmp, size, offset); + if(debug){ + if(strncmp(data, tmp,size) != 0){ + printf("Error\n"); + printf("%.*s", size, data); + printf("\n"); + printf("%.*s", size, tmp); + } + } + free(tmp); + /////////////////////////////// closeAcccess(&request); return writtenBytes; } @@ -436,28 +425,40 @@ int writeToBigBlock(unsigned long bigBlockId,const char *data, off_t offset,size createBigBlock(bigBlockId); } int firstSmallBlock = getSmallBlockId(offset); - int lastSmallBlock = getSmallBlockId(offset+size); + int lastSmallBlock = getSmallBlockId(offset+size-1); if(firstSmallBlock>255||lastSmallBlock>255){ printf("Error SmallBLock > 255"); } lseek(fh,filePointers[bigBlockId],SEEK_SET); int blockState[8] ; read(fh,&blockState,sizeof(int)*8); - lseek(fh,filePointers[bigBlockId]+4096+offset,SEEK_SET); + //If not on Block border and don't have this block, get the data. if((offset % 4096 !=0 )&&(!TestBit(blockState,firstSmallBlock))){ - char *startData = malloc(offset % 4096); - off_t tmp =offset+(bigBlockId*(4096*256)); - tmp = tmp - (tmp % 4096); - image_read_internal(startData, offset % 4096, tmp); + size_t sizeToPrepend = offset % 4096; + lseek(fh,filePointers[bigBlockId]+4096+offset-sizeToPrepend,SEEK_SET); + char *startData = calloc(sizeToPrepend,1); - write(fh,startData,offset % 4096); + off_t offsetToPrepend =offset+(bigBlockId*(4096*256))-sizeToPrepend; + // TODO CHECK THIS + //offsetToPrepend = offsetToPrepend - (offsetToPrepend % 4096); + + + if((((uint64_t)offsetToPrepend)+((uint64_t)sizeToPrepend)) > remoteImageSize ){ + sizeToPrepend = remoteImageSize-offsetToPrepend; + } + if(((uint64_t)offsetToPrepend)< remoteImageSize){ + image_read_internal(startData,sizeToPrepend, offsetToPrepend); + } + + write(fh,startData,(offset % 4096)); free(startData); + } else{ + lseek(fh,filePointers[bigBlockId]+4096+offset,SEEK_SET); } - writtenBytes +=(int)write(fh,data,size); //If not on Block border and don't have this block, get the data. if(((offset+size) % 4096 !=0 )&&(!TestBit(blockState,lastSmallBlock))){ @@ -470,41 +471,18 @@ int writeToBigBlock(unsigned long bigBlockId,const char *data, off_t offset,size if((((size_t)offsetToAppend)+sizeToAppend) > remoteImageSize ){ - sizeToAppend = remoteImageSize-offsetToAppend; } if(((size_t)offsetToAppend)< remoteImageSize){ image_read_internal(startData,sizeToAppend, offsetToAppend); } //lseek(fh,filePointers[bigBlockId]+4096+offset,SEEK_SET); - write(fh,startData,sizeToAppend); + write(fh,startData,(4096-((((size_t)offset)+size)% 4096))); free(startData); - /* - //size_t tmpSize = (4096-((offset+size) % 4096)); - size_t tmpSize = (((size_t)offset)+size); - tmpSize= tmpSize %4096; - tmpSize = 4096-tmpSize; - - off_t tmp =(offset+size-tmpSize)+(bigBlockId*(4096*256)); - - - if(((size_t)tmp) < remoteImageSize ){ - char *startData = calloc(tmpSize,1); - size_t remoteReadSize = tmpSize; - if ( (tmp + tmpSize) > remoteImageSize ) { - remoteReadSize = remoteImageSize - tmp; - - } - image_read_internal(startData,tmpSize, remoteReadSize); - lseek(fh,filePointers[bigBlockId]+4096+offset,SEEK_SET); - write(fh,startData,tmpSize); - free(startData); - } - */ } - for (long i = firstSmallBlock; i < lastSmallBlock;i++ ){ + for (long i = firstSmallBlock; i <= lastSmallBlock;i++ ){ SetBit(blockState,i); } lseek(fh,filePointers[bigBlockId],SEEK_SET); @@ -522,7 +500,8 @@ int cow_read(char *buf, size_t size, off_t offset) { unsigned long bigBlockStartId = (offset)/(4096*256); - unsigned long bigBlockEndId = (offset+size)/(4096*256); + //TODO CHECK THIS + unsigned long bigBlockEndId = (offset+size-1)/(4096*256); unsigned long bigBlockId = bigBlockStartId; cow_request request =getAccess(offset,size); size_t bigBlockStart = (bigBlockId*(4096*256)); @@ -546,7 +525,6 @@ int cow_read(char *buf, size_t size, off_t offset) int readBigBlock(long bigBlockId ,char *buf,size_t size, off_t offset){ - // If block isn't local if(filePointers[bigBlockId] == 0){ return image_read_internal(buf, size, (offset+(bigBlockId*(4096*256)))); @@ -569,34 +547,73 @@ int readBigBlock(long bigBlockId ,char *buf,size_t size, off_t offset){ if(block >255){ printf("ERROR SmallBlack id > 255"); } + } + off_t startOffset = offset+readBytes; + size_t sizeToRead = ((block+1)*4096)-startOffset; + if(sizeToRead>size-readBytes){ + sizeToRead= size-readBytes; + } + size_t sizeToRemoteRead = sizeToRead; + if(((uint64_t)sizeToRead)+((uint64_t)startOffset)>remoteImageSize){ + if(((uint64_t)startOffset)>remoteImageSize){ + sizeToRemoteRead = 0; + }else{ + sizeToRemoteRead =((size_t)remoteImageSize)-startOffset; + } + } + startOffset = startOffset+(bigBlockId*4096*256); + if(sizeToRemoteRead > 0){ + readBytes +=image_read_internal(curBuf, sizeToRemoteRead,startOffset); } + if(readBytes < sizeToRead){ + for(int i =((int)readBytes); readBytes < sizeToRead; i++){ + curBuf[i] = 0; + readBytes++; + } + } + curBuf +=sizeToRead; + /* off_t startOffset = startBlock*4096; if(startOffset < offset){ startOffset = offset; } + + size_t sizeToRead = ((block+1)*4096)-startOffset; if(sizeToRead>size-readBytes){ sizeToRead= size-readBytes; } size_t sizeToRemoteRead = sizeToRead; - if(sizeToRead+startOffset>remoteImageSize){ - sizeToRemoteRead = remoteImageSize-startOffset; + + + if(((uint64_t)sizeToRead)+((uint64_t)startOffset)>remoteImageSize){ + if(((uint64_t)startOffset)>remoteImageSize){ + sizeToRemoteRead = 0; + }else{ + sizeToRemoteRead =((size_t)remoteImageSize)-startOffset; + } } //request data over network //TODO Check if offset compution is correct startOffset = startOffset+(bigBlockId*4096*256); - readBytes +=image_read_internal(curBuf, sizeToRemoteRead, startOffset); + if(sizeToRemoteRead > 0){ + readBytes +=image_read_internal(curBuf, sizeToRemoteRead,startOffset); + } + // If on File End fill up rest with 0 - if(sizeToRead+startOffset>remoteImageSize){ + if(((uint64_t)sizeToRead)+((uint64_t)startOffset)>remoteImageSize){ for(size_t i = 0; i<(sizeToRead-sizeToRemoteRead); i++){ curBuf[i] = 0; } + readBytes+=(sizeToRead-sizeToRemoteRead); } + curBuf +=sizeToRead; + */ }else{ while(TestBit(blockState,block)&& block!= endBlock){ @@ -605,6 +622,19 @@ int readBigBlock(long bigBlockId ,char *buf,size_t size, off_t offset){ printf("ERROR SmallBlack id > 255"); } } + + + off_t startOffset = offset+readBytes; + + size_t sizeToRead = ((block+1)*4096)-startOffset; + if(sizeToRead>size-readBytes){ + sizeToRead= size-readBytes; + } + //read Data local + lseek(fh,filePointers[bigBlockId]+4096+startOffset,SEEK_SET); + readBytes += read(fh,curBuf,sizeToRead); + curBuf +=readBytes; + /* off_t startOffset = startBlock*4096; if(startOffset < offset){ startOffset = offset; @@ -617,6 +647,7 @@ int readBigBlock(long bigBlockId ,char *buf,size_t size, off_t offset){ lseek(fh,filePointers[bigBlockId]+4096+startOffset,SEEK_SET); readBytes += read(fh,curBuf,sizeToRead); curBuf +=readBytes; + */ } } -- cgit v1.2.3-55-g7522