From 3e992e066c6aeb3972758a2304cccbdba024c31e Mon Sep 17 00:00:00 2001 From: sr Date: Sat, 1 Sep 2012 18:08:27 +0200 Subject: [KERNEL] Tell server whether we're a client or a server in proxy mode [KERNEL] SysFS: Return empty string for current_server if not connected [SERVER] Code refactoring --- src/client/client.c | 2 +- src/config.h | 6 +++--- src/kernel/blk.c | 2 +- src/kernel/dnbd3.h | 2 +- src/kernel/net.c | 12 ++++++----- src/kernel/sysfs.c | 4 +++- src/serialize.c | 12 +++++++++++ src/serialize.h | 12 +++++++++-- src/server/ipc.c | 32 +++++++++++++++++---------- src/server/net.c | 8 ++++--- src/server/server.c | 16 +++++++------- src/server/server.h | 13 ++++++++++- src/server/utils.c | 62 +++++++++++++++++++++++++++-------------------------- src/types.h | 8 +++---- 14 files changed, 119 insertions(+), 72 deletions(-) (limited to 'src') diff --git a/src/client/client.c b/src/client/client.c index bb6b81e..81a727f 100644 --- a/src/client/client.c +++ b/src/client/client.c @@ -96,7 +96,7 @@ int main(int argc, char *argv[]) msg.port = htons(PORT); msg.addrtype = 0; msg.imgname = NULL; - msg.mode = DEVICE_MODE_CLIENT; + msg.is_server = FALSE; int opt = 0; int longIndex = 0; diff --git a/src/config.h b/src/config.h index 688b51c..ff638d2 100644 --- a/src/config.h +++ b/src/config.h @@ -28,11 +28,11 @@ // Protocol version should be increased whenever new features/messages are added, // so either the client or server can run in compatibility mode, or they can // cancel the connection right away if the protocol has changed too much -#define PROTOCOL_VERSION 1 +#define PROTOCOL_VERSION 2 // Which is the minimum protocol version the server expects from the client -#define MIN_SUPPORTED_CLIENT 1 +#define MIN_SUPPORTED_CLIENT 2 // Which is the minimum protocol version the client expects from the server -#define MIN_SUPPORTED_SERVER 1 +#define MIN_SUPPORTED_SERVER 2 // No payload allowed exceeding this many bytes (actual data from client->server is not affected by this limit!) #define MAX_PAYLOAD 1000 diff --git a/src/kernel/blk.c b/src/kernel/blk.c index 012471d..35252c4 100644 --- a/src/kernel/blk.c +++ b/src/kernel/blk.c @@ -147,7 +147,7 @@ int dnbd3_blk_ioctl(struct block_device *bdev, fmode_t mode, unsigned int cmd, u memcpy(&dev->initial_server, &dev->cur_server, sizeof(dev->initial_server)); dev->imgname = imgname; dev->rid = msg->rid; - dev->mode = msg->mode; + dev->is_server = msg->is_server; blk_queue->backing_dev_info.ra_pages = (msg->read_ahead_kb * 1024) / PAGE_CACHE_SIZE; if (dnbd3_net_connect(dev) == 0) { diff --git a/src/kernel/dnbd3.h b/src/kernel/dnbd3.h index 6d1bd17..00cc676 100644 --- a/src/kernel/dnbd3.h +++ b/src/kernel/dnbd3.h @@ -61,7 +61,7 @@ typedef struct dnbd3_server_t alt_servers[NUMBER_SERVERS]; // array of alt servers int new_servers_num; // number of new alt servers that are waiting to be copied to above array dnbd3_server_entry_t new_servers[NUMBER_SERVERS]; // pending new alt servers - uint8_t discover, panic, disconnecting, mode, update_available, panic_count; + uint8_t discover, panic, disconnecting, is_server, update_available, panic_count; uint16_t rid, heartbeat_count; uint64_t reported_size; // server switch diff --git a/src/kernel/net.c b/src/kernel/net.c index 475ae63..4b4da6f 100644 --- a/src/kernel/net.c +++ b/src/kernel/net.c @@ -134,7 +134,7 @@ int dnbd3_net_connect(dnbd3_device_t *dev) // Forget all known alt servers memset(dev->alt_servers, 0, sizeof(dev->alt_servers[0])*NUMBER_SERVERS); memcpy(dev->alt_servers, &dev->initial_server, sizeof(dev->alt_servers[0])); - if (dev->mode == DEVICE_MODE_CLIENT) + if (!dev->is_server) { req1 = kmalloc(sizeof(*req1), GFP_ATOMIC); if (!req1) @@ -181,6 +181,7 @@ int dnbd3_net_connect(dnbd3_device_t *dev) serializer_put_uint16(&dev->payload_buffer, PROTOCOL_VERSION); serializer_put_string(&dev->payload_buffer, dev->imgname); serializer_put_uint16(&dev->payload_buffer, dev->rid); + serializer_put_uint8(&dev->payload_buffer, dev->is_server); iov[1].iov_base = &dev->payload_buffer; iov[1].iov_len = serializer_get_written_length(&dev->payload_buffer); if (kernel_sendmsg(dev->sock, &msg, iov, 2, sizeof(dnbd3_request) + iov[1].iov_len) != sizeof(dnbd3_request) + iov[1].iov_len) @@ -407,7 +408,7 @@ int dnbd3_net_discover(void *data) debug_dev("FATAL: Kmalloc failed (discover)"); return -1; } - payload = (serialized_buffer_t*)buf; + payload = (serialized_buffer_t*)buf; // Reuse this buffer to save kernel mem dnbd3_request.magic = dnbd3_packet_magic; @@ -497,6 +498,7 @@ int dnbd3_net_discover(void *data) serializer_put_uint16(payload, PROTOCOL_VERSION); serializer_put_string(payload, dev->imgname); serializer_put_uint16(payload, dev->rid); + serializer_put_uint8(payload, 1); // Pretent we're a proxy here to prevent the server from updating the atime iov[1].iov_base = payload; dnbd3_request.size = iov[1].iov_len = serializer_get_written_length(payload); if (kernel_sendmsg(sock, &msg, iov, 2, sizeof(dnbd3_request) + iov[1].iov_len) != sizeof(dnbd3_request) + iov[1].iov_len) @@ -656,8 +658,8 @@ int dnbd3_net_discover(void *data) continue; } - // take server with lowest rtt - if (ready && best_server != current_server + // take server with lowest rtt (only if in client mode) + if (!dev->is_server && ready && best_server != current_server && RTT_THRESHOLD_FACTOR(dev->cur_rtt) > best_rtt) { printk("INFO: Server %d on %s is faster (%lluµs vs. %lluµs)\n", best_server, dev->disk->disk_name, (unsigned long long)best_rtt, (unsigned long long)dev->cur_rtt); @@ -873,7 +875,7 @@ int dnbd3_net_receive(void *data) continue; case CMD_GET_SERVERS: - if (dev->mode == DEVICE_MODE_PROXY || !is_same_server(&dev->cur_server, &dev->initial_server)) + if (dev->is_server || !is_same_server(&dev->cur_server, &dev->initial_server)) { // If not connected to initial server, or device is in proxy mode, ignore this message remaining = dnbd3_reply.size; goto clear_remaining_payload; diff --git a/src/kernel/sysfs.c b/src/kernel/sysfs.c index ff3b5f9..4c903ea 100644 --- a/src/kernel/sysfs.c +++ b/src/kernel/sysfs.c @@ -31,7 +31,9 @@ ssize_t show_cur_server_addr(char *buf, dnbd3_device_t *dev) { if (dev->cur_server.hostaddrtype == AF_INET) return MIN(snprintf(buf, PAGE_SIZE, "%pI4,%d\n", dev->cur_server.hostaddr, (int)ntohs(dev->cur_server.port)), PAGE_SIZE); - return MIN(snprintf(buf, PAGE_SIZE, "%pI6,%d\n", dev->cur_server.hostaddr, (int)ntohs(dev->cur_server.port)), PAGE_SIZE); + else if (dev->cur_server.hostaddrtype == AF_INET6) + return MIN(snprintf(buf, PAGE_SIZE, "%pI6,%d\n", dev->cur_server.hostaddr, (int)ntohs(dev->cur_server.port)), PAGE_SIZE); + return snprintf(buf, PAGE_SIZE, ""); } ssize_t show_cur_server_rtt(char *buf, dnbd3_device_t *dev) diff --git a/src/serialize.c b/src/serialize.c index ce27c8d..932df14 100644 --- a/src/serialize.c +++ b/src/serialize.c @@ -17,6 +17,12 @@ void serializer_reset_write(serialized_buffer_t *buffer) buffer->buffer_pointer = buffer->buffer; } +uint8_t serializer_get_uint8(serialized_buffer_t *buffer) +{ + if (buffer->buffer_pointer + 1 > buffer->buffer_end) return 0; + return (uint8_t)*buffer->buffer_pointer++; +} + uint16_t serializer_get_uint16(serialized_buffer_t *buffer) { uint16_t ret; @@ -45,6 +51,12 @@ char* serializer_get_string(serialized_buffer_t *buffer) return start; } +void serializer_put_uint8(serialized_buffer_t *buffer, uint8_t value) +{ + if (buffer->buffer_pointer + 1 > buffer->buffer_end) return; + *buffer->buffer_pointer++ = (char)value; +} + void serializer_put_uint16(serialized_buffer_t *buffer, uint16_t value) { if (buffer->buffer_pointer + 2 > buffer->buffer_end) return; diff --git a/src/serialize.h b/src/serialize.h index eeb3b26..970ffc2 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -14,18 +14,26 @@ void serializer_reset_read(serialized_buffer_t *buffer, size_t data_len); void serializer_reset_write(serialized_buffer_t *buffer); +ssize_t serializer_get_written_length(serialized_buffer_t *buffer); + +// + +uint8_t serializer_get_uint8(serialized_buffer_t *buffer); + uint16_t serializer_get_uint16(serialized_buffer_t *buffer); uint64_t serializer_get_uint64(serialized_buffer_t *buffer); char* serializer_get_string(serialized_buffer_t *buffer); +// + +void serializer_put_uint8(serialized_buffer_t *buffer, uint16_t value); + void serializer_put_uint16(serialized_buffer_t *buffer, uint16_t value); void serializer_put_uint64(serialized_buffer_t *buffer, uint64_t value); void serializer_put_string(serialized_buffer_t *buffer, char *value); -ssize_t serializer_get_written_length(serialized_buffer_t *buffer); - #endif diff --git a/src/server/ipc.c b/src/server/ipc.c index ebc907b..37ca36f 100644 --- a/src/server/ipc.c +++ b/src/server/ipc.c @@ -364,7 +364,8 @@ static int ipc_receive(int client_sock) GSList *iterator = NULL; struct tm *timeinfo; - char time_buff[64], rid[20], ipaddr[100]; +#define STRBUFLEN 100 + char strbuffer[STRBUFLEN]; dnbd3_ipc_t header; @@ -428,18 +429,27 @@ static int ipc_receive(int client_sock) for (iterator = _dnbd3_images; iterator; iterator = iterator->next) { const dnbd3_image_t *image = iterator->data; - sprintf(rid, "%d", image->rid); - timeinfo = localtime(&image->atime); - strftime(time_buff, 64, "%d.%m.%y %H:%M:%S", timeinfo); tmp_node = xmlNewNode(NULL, BAD_CAST "image"); if (tmp_node == NULL) goto get_info_reply_cleanup; xmlNewProp(tmp_node, BAD_CAST "name", BAD_CAST image->config_group); - xmlNewProp(tmp_node, BAD_CAST "atime", BAD_CAST time_buff); - xmlNewProp(tmp_node, BAD_CAST "rid", BAD_CAST rid); + timeinfo = localtime(&image->atime); + strftime(strbuffer, STRBUFLEN, "%d.%m.%y %H:%M:%S", timeinfo); + xmlNewProp(tmp_node, BAD_CAST "atime", BAD_CAST strbuffer); + sprintf(strbuffer, "%d", image->rid); + xmlNewProp(tmp_node, BAD_CAST "rid", BAD_CAST strbuffer); xmlNewProp(tmp_node, BAD_CAST "file", BAD_CAST image->file); - xmlNewProp(tmp_node, BAD_CAST "servers", BAD_CAST "???"); - xmlNewProp(tmp_node, BAD_CAST "cache", BAD_CAST image->cache_file); + xmlNewProp(tmp_node, BAD_CAST "servers", BAD_CAST "???"); // TODO + if (image->cache_file && image->cache_map) + { + xmlNewProp(tmp_node, BAD_CAST "cachefile", BAD_CAST image->cache_file); + int i, complete = 0, size = IMGSIZE_TO_MAPBYTES(image->filesize); + for (i = 0; i < size; ++i) + if (image->cache_map[i]) + complete += 100; + sprintf(strbuffer, "%d", complete / size); + xmlNewProp(tmp_node, BAD_CAST "cachefill", BAD_CAST image->cache_file); + } xmlAddChild(images_node, tmp_node); } // Clients @@ -455,9 +465,9 @@ static int ipc_receive(int client_sock) tmp_node = xmlNewNode(NULL, BAD_CAST "client"); if (tmp_node == NULL) goto get_info_reply_cleanup; - *ipaddr = '\0'; - inet_ntop(client->addrtype, client->ipaddr, ipaddr, 100); - xmlNewProp(tmp_node, BAD_CAST "ip", BAD_CAST ipaddr); + *strbuffer = '\0'; + inet_ntop(client->addrtype, client->ipaddr, strbuffer, STRBUFLEN); + xmlNewProp(tmp_node, BAD_CAST "ip", BAD_CAST strbuffer); xmlNewProp(tmp_node, BAD_CAST "file", BAD_CAST client->image->file); xmlAddChild(clients_node, tmp_node); } diff --git a/src/server/net.c b/src/server/net.c index b1a7b20..37ac0b4 100644 --- a/src/server/net.c +++ b/src/server/net.c @@ -160,6 +160,7 @@ void *dnbd3_handle_query(void *dnbd3_client) client_version = serializer_get_uint16(&payload); image_name = serializer_get_string(&payload); rid = serializer_get_uint16(&payload); + client->is_server = serializer_get_uint8(&payload); if (request.size < 3 || !image_name || client_version < MIN_SUPPORTED_CLIENT) { if (client_version < MIN_SUPPORTED_CLIENT) @@ -208,7 +209,8 @@ void *dnbd3_handle_query(void *dnbd3_client) else { client->image = image; - image->atime = time(NULL); // TODO: check if mutex is needed + if (!client->is_server) + image->atime = time(NULL); // TODO: check if mutex is needed if (image->cache_map && image->cache_file) image_cache = open(image->cache_file, O_RDWR); @@ -288,8 +290,8 @@ void *dnbd3_handle_query(void *dnbd3_client) // first make sure the whole requested part is in the local cache file while(cur_offset < last_offset) { - map_y = cur_offset >> 15; - map_x = (cur_offset >> 12) & 7; // mod 256 + map_y = cur_offset >> 15; // div 32768 + map_x = (cur_offset >> 12) & 7; // (X div 4096) mod 8 bit_mask = 0b00000001 << (map_x); cur_offset += 4096; diff --git a/src/server/server.c b/src/server/server.c index 28da0ae..1ca55fa 100644 --- a/src/server/server.c +++ b/src/server/server.c @@ -52,16 +52,16 @@ void dnbd3_print_help(char* argv_0) { printf("Usage: %s [OPTIONS]...\n", argv_0); printf("Start the DNBD3 server\n"); - printf("-f or --file \t\t Configuration file (default /etc/dnbd3-server.conf)\n"); + printf("-f or --file Configuration file (default /etc/dnbd3-server.conf)\n"); #ifdef _DEBUG - printf("-d or --delay \t\t Add a fake network delay of X µs\n"); + printf("-d or --delay Add a fake network delay of X µs\n"); #endif - printf("-n or --nodaemon \t Start server in foreground\n"); - printf("-r or --reload \t\t Reload configuration file\n"); - printf("-s or --stop \t\t Stop running dnbd3-server\n"); - printf("-i or --info \t\t Print connected clients and used images\n"); - printf("-H or --help \t\t Show this help text and quit\n"); - printf("-V or --version \t Show version and quit\n"); + printf("-n or --nodaemon Start server in foreground\n"); + printf("-r or --reload Reload configuration file\n"); + printf("-s or --stop Stop running dnbd3-server\n"); + printf("-i or --info Print connected clients and used images\n"); + printf("-H or --help Show this help text and quit\n"); + printf("-V or --version Show version and quit\n"); exit(0); } diff --git a/src/server/server.h b/src/server/server.h index 90aae5c..832777e 100644 --- a/src/server/server.h +++ b/src/server/server.h @@ -28,6 +28,15 @@ #include "../config.h" #include "../types.h" +// one byte in the map covers 8 4kib blocks, so 32kib per byte +// "+ (1 << 15) - 1" is required to account for the last bit of +// the image that is smaller than 32kib +// this would be the case whenever the image file size is not a +// multiple of 32kib (= the number of blocks is not divisible by 8) +// ie: if the image is 49152 bytes and you do 49152 >> 15 you get 1, +// but you actually need 2 bytes to have a complete cache map +#define IMGSIZE_TO_MAPBYTES(bytes) ((int)(((bytes) + (1 << 15) - 1) >> 15)) + typedef struct { char *config_group; // exact name of group in config file that represents this image @@ -42,13 +51,15 @@ typedef struct char working; // whether this image is considered working. local images are "working" if the local file exists, proxied images have to have at least one working upstream server or a complete local cache file time_t delete_soft; // unixtime telling when this image should be deleted. if there are still clients using this image it weill be kept, but new clients requesting the image will be rejected. 0 = never time_t delete_hard; // unixtime telling when this image should be deleted, no matter if there are still clients connected. 0 = never + uint8_t relayed; // TRUE if relayed from other server (needs dnbd3 client module loaded) } dnbd3_image_t; typedef struct { int sock; uint8_t ipaddr[16]; - uint8_t addrtype; // ip version (AF_INET or AF_INET6) + uint8_t addrtype; // ip version (AF_INET or AF_INET6) + uint8_t is_server; // TRUE if a server in proxy mode, FALSE if real client pthread_t thread; dnbd3_image_t *image; } dnbd3_client_t; diff --git a/src/server/utils.c b/src/server/utils.c index 665d96d..391992d 100644 --- a/src/server/utils.c +++ b/src/server/utils.c @@ -438,9 +438,9 @@ static dnbd3_image_t *prepare_image(char *image_name, int rid, char *image_file, image->config_group = strdup(image_name); image->rid = rid; - const char relayed = (image_file == NULL || image_file == '\0'); + image->relayed = (image_file == NULL || image_file == '\0'); - if (relayed) // Image is relayed (this server acts as proxy) + if (image->relayed) // Image is relayed (this server acts as proxy) { if (strchr(image_name, '/') == NULL) { @@ -508,45 +508,47 @@ static dnbd3_image_t *prepare_image(char *image_name, int rid, char *image_file, } else if (image->filesize > 0) { - const size_t map_len_bytes = (image->filesize + (1 << 15) - 1) >> 15; + const size_t map_len_bytes = IMGSIZE_TO_MAPBYTES(image->filesize); image->cache_map = calloc(map_len_bytes, sizeof(uint8_t)); // read cache map from file - // one byte in the map covers 8 4kib blocks, so 32kib per byte - // "+ (1 << 15) - 1" is required to account for the last bit of - // the image that is smaller than 32kib - // this would be the case whenever the image file size is not a - // multiple of 32kib (= the number of blocks is not divisible by 8) - // ie: if the image is 49152 bytes and you do 49152 >> 15 you get 1, - // but you actually need 2 bytes to have a complete cache map char tmp[strlen(image->cache_file) + 5]; strcpy(tmp, image->cache_file); strcat(tmp, ".map"); - fd = open(tmp, O_RDONLY); // TODO: Check if map file has expected size + fd = open(tmp, O_RDONLY); if (fd >= 0) { - read(fd, image->cache_map, map_len_bytes * sizeof(uint8_t)); - close(fd); - // If the whole image is cached, mark it as working right away without waiting for an upstream server - image->working = 1; - for (j = 0; j < map_len_bytes - 1; ++j) + const off_t size = lseek(fd, 0, SEEK_END); + if (size != map_len_bytes) { - if (image->cache_map[j] != 0xFF) + memlogf("[DEBUG] Cache-Map of %s is corrupted (%d != %d)", image_name, (int)size, (int)map_len_bytes); + } + else + { + lseek(fd, 0, SEEK_SET); + read(fd, image->cache_map, map_len_bytes); + // If the whole image is cached, mark it as working right away without waiting for an upstream server + image->working = 1; + for (j = 0; j < map_len_bytes - 1; ++j) { - image->working = 0; - break; + if (image->cache_map[j] != 0xFF) + { + image->working = 0; + break; + } } + const int blocks_in_last_byte = (image->filesize >> 12) & 7; + uint8_t last_byte = 0; + if (blocks_in_last_byte == 0) + last_byte = 0xFF; + else + for (j = 0; j < blocks_in_last_byte; ++j) + last_byte |= (1 << j); + if ((image->cache_map[map_len_bytes - 1] & last_byte) != last_byte) + image->working = 0; + else + memlogf("[INFO] Instantly publishing relayed image '%s' because the local cache copy is complete", image_name); } - const int blocks_in_last_byte = (image->filesize >> 12) & 7; - uint8_t last_byte = 0; - if (blocks_in_last_byte == 0) - last_byte = 0xFF; - else - for (j = 0; j < blocks_in_last_byte; ++j) - last_byte = (last_byte << 1) | 1; - if ((image->cache_map[map_len_bytes - 1] & last_byte) != last_byte) - image->working = 0; - else - memlogf("[INFO] Instantly publishing relayed image '%s' because the local cache copy is complete", image_name); + close(fd); } /* diff --git a/src/types.h b/src/types.h index a264288..ab3a2f0 100644 --- a/src/types.h +++ b/src/types.h @@ -64,16 +64,14 @@ typedef struct { uint16_t len; uint8_t addrtype; - uint8_t addr[16]; // network representation - uint16_t port; // network representation + uint8_t addr[16]; // network representation + uint16_t port; // network representation uint16_t imgnamelen; char *imgname; int rid; int read_ahead_kb; - uint8_t mode; // 0 = automatic (real client), 1 = manual control (proxy) + uint8_t is_server; // FALSE = automatic (real client), TRUE = manual control (proxy) } dnbd3_ioctl_t; -#define DEVICE_MODE_CLIENT 0 -#define DEVICE_MODE_PROXY 1 // network #define CMD_GET_BLOCK 1 -- cgit v1.2.3-55-g7522