From 7358f2786564e9a9960779e23ae36a26313f9bf3 Mon Sep 17 00:00:00 2001 From: Johann Latocha Date: Tue, 6 Mar 2012 19:27:21 +0100 Subject: [SERVER] Cache images on hdd --- src/kernel/blk.c | 4 ++++ src/server/net.c | 37 ++++++++++++++++++++++++++++++++++--- src/server/server.c | 20 ++++++++++++++++++++ src/server/server.h | 2 ++ src/server/utils.c | 45 +++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 105 insertions(+), 3 deletions(-) diff --git a/src/kernel/blk.c b/src/kernel/blk.c index 2b12b74..b975600 100644 --- a/src/kernel/blk.c +++ b/src/kernel/blk.c @@ -73,6 +73,10 @@ int dnbd3_blk_add_device(dnbd3_device_t *dev, int minor) blk_queue_logical_block_size(blk_queue, DNBD3_BLOCK_SIZE); blk_queue_physical_block_size(blk_queue, DNBD3_BLOCK_SIZE); + blk_queue_max_hw_sectors(blk_queue, DNBD3_BLOCK_SIZE/KERNEL_SECTOR_SIZE); + blk_queue_max_segment_size(blk_queue, DNBD3_BLOCK_SIZE); + blk_queue_max_segments(blk_queue, 1); + disk->queue = blk_queue; disk->private_data = dev; queue_flag_set_unlocked(QUEUE_FLAG_NONROT, disk->queue); diff --git a/src/server/net.c b/src/server/net.c index 7e9d788..1d47e92 100644 --- a/src/server/net.c +++ b/src/server/net.c @@ -41,11 +41,14 @@ void *dnbd3_handle_query(void *dnbd3_client) dnbd3_reply_t reply; dnbd3_image_t *image = NULL; - int image_file = -1; + int image_file, image_cache = -1; struct in_addr alt_server; int i = 0; + uint64_t map_y; + char map_x, bit_mask; + while (recv(client->sock, &request, sizeof(dnbd3_request_t), MSG_WAITALL) > 0) { reply.cmd = request.cmd; @@ -89,6 +92,10 @@ void *dnbd3_handle_query(void *dnbd3_client) image_file = open(image->file, O_RDONLY); client->image = image; image->atime = time(NULL); // TODO: check if mutex is needed + + if (image->cache_file) + image_cache = open(image->cache_file, O_RDWR); + break; case CMD_GET_BLOCK: @@ -98,8 +105,31 @@ void *dnbd3_handle_query(void *dnbd3_client) reply.size = request.size; send(client->sock, (char *) &reply, sizeof(dnbd3_reply_t), 0); - if (sendfile(client->sock, image_file, (off_t *) &request.offset, request.size) < 0) - printf("ERROR: Sendfile failed\n"); + // caching is off + if (!image->cache_file) + { + if (sendfile(client->sock, image_file, (off_t *) &request.offset, request.size) < 0) + printf("ERROR: Sendfile failed (sock)\n"); + + break; + } + + map_y = request.offset >> 15; + map_x = (request.offset >> 12) & 7; // mod 8 + bit_mask = 0b00000001 << (map_x); + + if ((image->cache_map[map_y] & bit_mask) == 0) // cache miss + { + uint64_t tmp = request.offset; + lseek(image_cache, tmp, SEEK_SET); + if (sendfile(image_cache, image_file, (off_t *) &tmp, request.size) < 0) + printf("ERROR: Sendfile failed (cache)\n"); + + image->cache_map[map_y] |= bit_mask; // set 1 in cache map + } + + if (sendfile(client->sock, image_cache, (off_t *) &request.offset, request.size) < 0) + printf("ERROR: Sendfile failed (net)\n"); break; @@ -121,6 +151,7 @@ void *dnbd3_handle_query(void *dnbd3_client) } close(client->sock); close(image_file); + close(image_cache); pthread_spin_lock(&_spinlock); _dnbd3_clients = g_slist_remove(_dnbd3_clients, client); pthread_spin_unlock(&_spinlock); diff --git a/src/server/server.c b/src/server/server.c index 48dd545..c77fc7e 100644 --- a/src/server/server.c +++ b/src/server/server.c @@ -25,6 +25,7 @@ #include #include #include +#include #include "../types.h" #include "../version.h" @@ -64,6 +65,7 @@ void dnbd3_print_version() void dnbd3_cleanup() { + int i, fd; printf("INFO: Cleanup...\n"); pthread_spin_lock(&_spinlock); @@ -75,6 +77,24 @@ void dnbd3_cleanup() pthread_join(*client->thread, NULL); } g_slist_free(_dnbd3_clients); + + // save cache maps to files + for (i = 0; i < _num_images; i++) + { + if (_images[i].cache_file) + { + char tmp[strlen(_images[i].cache_file)+4]; + strcpy(tmp, _images[i].cache_file); + strcat(tmp, ".map"); + fd = open(tmp, O_WRONLY | O_CREAT, S_IRUSR | S_IWUSR); + + if (fd > 0) + write(fd, _images[i].cache_map, (_images[i].filesize >> 15) * sizeof(char)); + + close(fd); + } + } + pthread_spin_unlock(&_spinlock); close(_sock); diff --git a/src/server/server.h b/src/server/server.h index e73c171..b21b92c 100644 --- a/src/server/server.h +++ b/src/server/server.h @@ -37,6 +37,8 @@ typedef struct int vid; int rid; time_t atime; + char *cache_map; + char *cache_file; } dnbd3_image_t; typedef struct diff --git a/src/server/utils.c b/src/server/utils.c index f2f5e35..54ebda9 100644 --- a/src/server/utils.c +++ b/src/server/utils.c @@ -22,6 +22,7 @@ #include #include #include +#include #include "server.h" #include "utils.h" @@ -49,6 +50,7 @@ void dnbd3_load_config(char *file) _images[i].servers = g_key_file_get_string_list(gkf, groups[i], "servers", &_images[i].num_servers, NULL); _images[i].vid = g_key_file_get_integer(gkf, groups[i], "vid", NULL); _images[i].rid = g_key_file_get_integer(gkf, groups[i], "rid", NULL); + _images[i].cache_file = g_key_file_get_string(gkf, groups[i], "cache", NULL); _images[i].atime = 0; if (_images[i].num_servers > NUMBER_SERVERS) @@ -61,6 +63,30 @@ void dnbd3_load_config(char *file) printf("ERROR: Image not found: %s\n", _images[i].file); close(fd); + + if (_images[i].cache_file) + { + // read cache map from file + _images[i].cache_map = calloc(_images[i].filesize >> 15, sizeof(char)); + memset(_images[i].cache_map, 0, (_images[i].filesize >> 15) * sizeof(char)); + char tmp[strlen(_images[i].cache_file)+4]; + strcpy(tmp, _images[i].cache_file); + strcat(tmp, ".map"); + fd = open(tmp, O_RDONLY); + if (fd > 0) + read(fd, _images[i].cache_map, (_images[i].filesize >> 15) * sizeof(char)); + close(fd); + + // open cache file + fd = open(_images[i].cache_file, O_WRONLY | O_CREAT, S_IRUSR | S_IWUSR); + if (fd < 1) + printf("ERROR: Could't create cache file\n"); + + if (_images[i].filesize != lseek(fd, 0, SEEK_END)) + fallocate(fd, 0, 0, _images[i].filesize); + + close(fd); + } } g_strfreev(groups); @@ -69,6 +95,7 @@ void dnbd3_load_config(char *file) void dnbd3_reload_config(char* config_file_name) { + int i, fd; pthread_spin_lock(&_spinlock); GSList *iterator = NULL; for (iterator = _dnbd3_clients; iterator; iterator = iterator->next) @@ -77,6 +104,24 @@ void dnbd3_reload_config(char* config_file_name) pthread_spin_lock(&client->spinlock); client->image = NULL; } + + // save cache maps + for (i = 0; i < _num_images; i++) + { + if (_images[i].cache_file) + { + char tmp[strlen(_images[i].cache_file)+4]; + strcpy(tmp, _images[i].cache_file); + strcat(tmp, ".map"); + fd = open(tmp, O_WRONLY | O_CREAT, S_IRUSR | S_IWUSR); + + if (fd > 0) + write(fd, _images[i].cache_map, (_images[i].filesize >> 15) * sizeof(char)); + + close(fd); + } + } + _num_images = 0; free(_images); dnbd3_load_config(config_file_name); -- cgit v1.2.3-55-g7522