[SERVER] Cache images on hdd

5 files changed, 105 insertions, 3 deletions

diff --git a/src/kernel/blk.c b/src/kernel/blk.c
index 2b12b74..b975600 100644
--- a/src/kernel/blk.c
+++ b/src/kernel/blk.c
@@ -73,6 +73,10 @@ int dnbd3_blk_add_device(dnbd3_device_t *dev, int minor)
     blk_queue_logical_block_size(blk_queue, DNBD3_BLOCK_SIZE);
     blk_queue_physical_block_size(blk_queue, DNBD3_BLOCK_SIZE);
 
+    blk_queue_max_hw_sectors(blk_queue, DNBD3_BLOCK_SIZE/KERNEL_SECTOR_SIZE);
+    blk_queue_max_segment_size(blk_queue, DNBD3_BLOCK_SIZE);
+    blk_queue_max_segments(blk_queue, 1);
+
     disk->queue = blk_queue;
     disk->private_data = dev;
     queue_flag_set_unlocked(QUEUE_FLAG_NONROT, disk->queue);
diff --git a/src/server/net.c b/src/server/net.c
index 7e9d788..1d47e92 100644
--- a/src/server/net.c
+++ b/src/server/net.c
@@ -41,11 +41,14 @@ void *dnbd3_handle_query(void *dnbd3_client)
     dnbd3_reply_t reply;
 
     dnbd3_image_t *image = NULL;
-    int image_file = -1;
+    int image_file, image_cache = -1;
 
     struct in_addr alt_server;
     int i = 0;
 
+    uint64_t map_y;
+    char map_x, bit_mask;
+
     while (recv(client->sock, &request, sizeof(dnbd3_request_t), MSG_WAITALL) > 0)
     {
         reply.cmd = request.cmd;
@@ -89,6 +92,10 @@ void *dnbd3_handle_query(void *dnbd3_client)
             image_file = open(image->file, O_RDONLY);
             client->image = image;
             image->atime = time(NULL); // TODO: check if mutex is needed
+
+            if (image->cache_file)
+                image_cache = open(image->cache_file, O_RDWR);
+
             break;
 
         case CMD_GET_BLOCK:
@@ -98,8 +105,31 @@ void *dnbd3_handle_query(void *dnbd3_client)
             reply.size = request.size;
             send(client->sock, (char *) &reply, sizeof(dnbd3_reply_t), 0);
 
-            if (sendfile(client->sock, image_file, (off_t *) &request.offset, request.size) < 0)
-                printf("ERROR: Sendfile failed\n");
+            // caching is off
+            if (!image->cache_file)
+            {
+                if (sendfile(client->sock, image_file, (off_t *) &request.offset, request.size) < 0)
+                    printf("ERROR: Sendfile failed (sock)\n");
+
+                break;
+            }
+
+            map_y = request.offset >> 15;
+            map_x = (request.offset >> 12) & 7; // mod 8
+            bit_mask = 0b00000001 << (map_x);
+
+            if ((image->cache_map[map_y] & bit_mask) == 0) // cache miss
+            {
+                uint64_t tmp = request.offset;
+                lseek(image_cache, tmp, SEEK_SET);
+                if (sendfile(image_cache, image_file, (off_t *) &tmp, request.size) < 0)
+                    printf("ERROR: Sendfile failed (cache)\n");
+
+                image->cache_map[map_y] |= bit_mask; // set 1 in cache map
+            }
+
+            if (sendfile(client->sock, image_cache, (off_t *) &request.offset, request.size) < 0)
+                printf("ERROR: Sendfile failed (net)\n");
 
             break;
 
@@ -121,6 +151,7 @@ void *dnbd3_handle_query(void *dnbd3_client)
     }
     close(client->sock);
     close(image_file);
+    close(image_cache);
     pthread_spin_lock(&_spinlock);
     _dnbd3_clients = g_slist_remove(_dnbd3_clients, client);
     pthread_spin_unlock(&_spinlock);
diff --git a/src/server/server.c b/src/server/server.c
index 48dd545..c77fc7e 100644
--- a/src/server/server.c
+++ b/src/server/server.c
@@ -25,6 +25,7 @@
 #include <getopt.h>
 #include <pthread.h>
 #include <string.h>
+#include <fcntl.h>
 
 #include "../types.h"
 #include "../version.h"
@@ -64,6 +65,7 @@ void dnbd3_print_version()
 
 void dnbd3_cleanup()
 {
+    int i, fd;
     printf("INFO: Cleanup...\n");
 
     pthread_spin_lock(&_spinlock);
@@ -75,6 +77,24 @@ void dnbd3_cleanup()
         pthread_join(*client->thread, NULL);
     }
     g_slist_free(_dnbd3_clients);
+
+    // save cache maps to files
+    for (i = 0; i < _num_images; i++)
+    {
+        if (_images[i].cache_file)
+        {
+            char tmp[strlen(_images[i].cache_file)+4];
+            strcpy(tmp, _images[i].cache_file);
+            strcat(tmp, ".map");
+            fd = open(tmp, O_WRONLY | O_CREAT, S_IRUSR | S_IWUSR);
+
+            if (fd > 0)
+                write(fd, _images[i].cache_map,  (_images[i].filesize >> 15) * sizeof(char));
+
+            close(fd);
+        }
+    }
+
     pthread_spin_unlock(&_spinlock);
 
     close(_sock);
diff --git a/src/server/server.h b/src/server/server.h
index e73c171..b21b92c 100644
--- a/src/server/server.h
+++ b/src/server/server.h
@@ -37,6 +37,8 @@ typedef struct
     int vid;
     int rid;
     time_t atime;
+    char *cache_map;
+    char *cache_file;
 } dnbd3_image_t;
 
 typedef struct
diff --git a/src/server/utils.c b/src/server/utils.c
index f2f5e35..54ebda9 100644
--- a/src/server/utils.c
+++ b/src/server/utils.c
@@ -22,6 +22,7 @@
 #include <fcntl.h>
 #include <sys/stat.h>
 #include <pthread.h>
+#include <string.h>
 
 #include "server.h"
 #include "utils.h"
@@ -49,6 +50,7 @@ void dnbd3_load_config(char *file)
         _images[i].servers = g_key_file_get_string_list(gkf, groups[i], "servers", &_images[i].num_servers, NULL);
         _images[i].vid = g_key_file_get_integer(gkf, groups[i], "vid", NULL);
         _images[i].rid = g_key_file_get_integer(gkf, groups[i], "rid", NULL);
+        _images[i].cache_file = g_key_file_get_string(gkf, groups[i], "cache", NULL);
         _images[i].atime = 0;
 
         if (_images[i].num_servers > NUMBER_SERVERS)
@@ -61,6 +63,30 @@ void dnbd3_load_config(char *file)
             printf("ERROR: Image not found: %s\n", _images[i].file);
 
         close(fd);
+
+        if (_images[i].cache_file)
+        {
+            // read cache map from file
+            _images[i].cache_map  = calloc(_images[i].filesize >> 15, sizeof(char));
+            memset(_images[i].cache_map, 0, (_images[i].filesize >> 15) * sizeof(char));
+            char tmp[strlen(_images[i].cache_file)+4];
+            strcpy(tmp, _images[i].cache_file);
+            strcat(tmp, ".map");
+            fd = open(tmp, O_RDONLY);
+            if (fd > 0)
+                read(fd, _images[i].cache_map, (_images[i].filesize >> 15) * sizeof(char));
+            close(fd);
+
+            // open cache file
+            fd = open(_images[i].cache_file, O_WRONLY | O_CREAT, S_IRUSR | S_IWUSR);
+            if (fd < 1)
+                printf("ERROR: Could't create cache file\n");
+
+            if (_images[i].filesize != lseek(fd, 0, SEEK_END))
+                fallocate(fd, 0, 0, _images[i].filesize);
+
+            close(fd);
+        }
     }
 
     g_strfreev(groups);
@@ -69,6 +95,7 @@ void dnbd3_load_config(char *file)
 
 void dnbd3_reload_config(char* config_file_name)
 {
+    int i, fd;
     pthread_spin_lock(&_spinlock);
     GSList *iterator = NULL;
     for (iterator = _dnbd3_clients; iterator; iterator = iterator->next)
@@ -77,6 +104,24 @@ void dnbd3_reload_config(char* config_file_name)
         pthread_spin_lock(&client->spinlock);
         client->image = NULL;
     }
+
+    // save cache maps
+    for (i = 0; i < _num_images; i++)
+    {
+        if (_images[i].cache_file)
+        {
+            char tmp[strlen(_images[i].cache_file)+4];
+            strcpy(tmp, _images[i].cache_file);
+            strcat(tmp, ".map");
+            fd = open(tmp, O_WRONLY | O_CREAT, S_IRUSR | S_IWUSR);
+
+            if (fd > 0)
+                write(fd, _images[i].cache_map,  (_images[i].filesize >> 15) * sizeof(char));
+
+            close(fd);
+        }
+    }
+
     _num_images = 0;
     free(_images);
     dnbd3_load_config(config_file_name);