From 09a25c819fc6784f2ac61a37fbe36b408d838cec Mon Sep 17 00:00:00 2001 From: sr Date: Tue, 9 Jul 2013 19:14:34 +0200 Subject: Rewrite in progres.... --- src/server/globals.c | 6 ++ src/server/globals.h | 14 +++ src/server/helper.h | 10 ++ src/server/image.c | 273 ++++++++++++++++++++++++++++++++++++++++++++++----- src/server/image.h | 10 ++ src/server/net.c | 1 - src/server/server.c | 171 +++++++++++++++++++------------- src/server/server.h | 24 +++-- 8 files changed, 405 insertions(+), 104 deletions(-) create mode 100644 src/server/globals.c create mode 100644 src/server/globals.h (limited to 'src/server') diff --git a/src/server/globals.c b/src/server/globals.c new file mode 100644 index 0000000..9919ee3 --- /dev/null +++ b/src/server/globals.c @@ -0,0 +1,6 @@ +#include "globals.h" +#include +#include + +char *_basePath = NULL; +int _vmdkLegacyMode = FALSE; diff --git a/src/server/globals.h b/src/server/globals.h new file mode 100644 index 0000000..ccdaa01 --- /dev/null +++ b/src/server/globals.h @@ -0,0 +1,14 @@ +#ifndef _GLOBALS_H_ +#define _GLOBALS_H_ + +/** + * Base directory where all images are stored in. Will always have a trailing slash + */ +extern char *_basePath; + +/** + * Whether or not simple *.vmdk files should be treated as revision 1 + */ +extern int _vmdkLegacyMode; + +#endif /* GLOBALS_H_ */ diff --git a/src/server/helper.h b/src/server/helper.h index 5145905..1df86e8 100644 --- a/src/server/helper.h +++ b/src/server/helper.h @@ -81,6 +81,16 @@ static inline int recv_data(int client_sock, void *buffer_out, int len) return 0; } +static inline int strend(char *string, char *suffix) +{ + if (string == NULL) return FALSE; + if (suffix == NULL || *suffix == '\0') return TRUE; + const size_t len1 = strlen(string); + const size_t len2 = strlen(suffix); + if (len2 > len1) return FALSE; + return strcmp(string + len1 - len2, suffix) == 0; +} + // one byte in the map covers 8 4kib blocks, so 32kib per byte // "+ (1 << 15) - 1" is required to account for the last bit of // the image that is smaller than 32kib diff --git a/src/server/image.c b/src/server/image.c index 8a2b943..7e28e95 100644 --- a/src/server/image.c +++ b/src/server/image.c @@ -2,37 +2,258 @@ #include #include +#include +#include +#include +#include +#include +#include +// ########################################## + +static void image_load_all(char *path); +static int image_try_load(char *base, char *path); + +// ########################################## + +/** + * Returns TRUE if the given image is complete + */ int image_is_complete(dnbd3_image_t *image) { - assert(image != NULL); - if (image->working && image->cache_map == NULL) { + assert( image != NULL ); + if ( image->working && image->cache_map == NULL ) { return TRUE; } - if (image->filesize == 0 || !image->working) { + if ( image->filesize == 0 || !image->working ) { return FALSE; } - int complete = TRUE, j; - const int map_len_bytes = IMGSIZE_TO_MAPBYTES(image->filesize); - for (j = 0; j < map_len_bytes - 1; ++j) - { - if (image->cache_map[j] != 0xFF) - { - complete = FALSE; - break; - } - } - if (complete) // Every block except the last one is complete - { // Last one might need extra treatment if it's not a full byte - const int blocks_in_last_byte = (image->filesize >> 12) & 7; - uint8_t last_byte = 0; - if (blocks_in_last_byte == 0) { - last_byte = 0xFF; - } else { - for (j = 0; j < blocks_in_last_byte; ++j) - last_byte |= (1 << j); - } - complete = ((image->cache_map[map_len_bytes - 1] & last_byte) == last_byte); - } - return complete; + int complete = TRUE, j; + const int map_len_bytes = IMGSIZE_TO_MAPBYTES( image->filesize ); + for (j = 0; j < map_len_bytes - 1; ++j) { + if ( image->cache_map[j] != 0xFF ) { + complete = FALSE; + break; + } + } + if ( complete ) // Every block except the last one is complete + { // Last one might need extra treatment if it's not a full byte + const int blocks_in_last_byte = (image->filesize >> 12) & 7; + uint8_t last_byte = 0; + if ( blocks_in_last_byte == 0 ) { + last_byte = 0xFF; + } else { + for (j = 0; j < blocks_in_last_byte; ++j) + last_byte |= (1 << j); + } + complete = ((image->cache_map[map_len_bytes - 1] & last_byte) == last_byte); + } + return complete; +} + +/** + * Saves the cache map of the given image. + * Return TRUE on success. + */ +int image_save_cache_map(dnbd3_image_t *image) +{ + if ( image == NULL ) return TRUE; + char mapfile[strlen( image->path ) + 4 + 1]; + int fd; + strcpy( mapfile, image->path ); + strcat( mapfile, ".map" ); + + fd = open( mapfile, O_WRONLY | O_CREAT, S_IRUSR | S_IWUSR ); + if ( fd < 0 ) return FALSE; + + write( fd, image->cache_map, ((image->filesize + (1 << 15) - 1) >> 15) * sizeof(char) ); + close( fd ); + + return TRUE; } + +/** + * Get an image by name+rid. This function increases a reference counter, + * so you HAVE TO CALL image_release for every image_get() call at some + * point... + * Locks on: _images_lock, _images[].lock + */ +dnbd3_image_t* image_get(char *name, uint16_t revision) +{ + int i; + dnbd3_image_t *candidate = NULL; + // Simple sanity check + const int len = strlen( name ); + if ( len == 0 || name[len - 1] == '/' || name[0] == '/' ) return NULL ; + // Always use lowercase name + strtolower( name ); + // Go through array + pthread_spin_lock( &_images_lock ); + for (i = 0; i < _num_images; ++i) { + dnbd3_image_t * const image = _images[i]; + if ( image == NULL ) continue; + if ( strcmp( image->lower_name, name ) == 0 && revision == image->rid ) { + candidate = image; + break; + } else if ( revision == 0 && (candidate == NULL || candidate->rid < image->rid) ) { + candidate = image; + } + } + + if ( candidate == NULL ) { + pthread_spin_unlock( &_images_lock ); + return NULL ; + } + + pthread_spin_lock( &candidate->lock ); + pthread_spin_unlock( &_images_lock ); + + if ( candidate == NULL ) return NULL ; // Not found + // Found, see if it works + struct stat st; + if ( !candidate->working || candidate->delete_soft < time( NULL ) || stat( candidate->path, &st ) < 0 ) { + candidate->working = FALSE; + pthread_spin_unlock( &candidate->lock ); + return NULL ; // Not working (anymore) + } + candidate->users++; + pthread_spin_unlock( &candidate->lock ); + return candidate; // Success :-) +} + +/** + * Release given image. This will merely decrease the reference counter of the image. + * Locks on: _images[].lock + */ +void image_release(dnbd3_image_t *image) +{ + assert( image != NULL ); + pthread_spin_lock( &image->lock ); + assert( image->users > 0 ); + image->users--; + pthread_spin_unlock( &image->lock ); +} + +void image_load_all() +{ + +} + +/** + * Load all images in the given path + */ +static int image_load_all(char *base, char *path) +{ +#define SUBDIR_LEN 120 + assert( path != NULL ); + assert( *path == '/' ); + struct dirent *entry; + DIR *dir = opendir( path ); + if ( dir == NULL ) { + memlogf( "[ERROR] Could not opendir '%s' for loading", path ); + return FALSE; + } + const int pathLen = strlen( path ); + const int len = pathLen + SUBDIR_LEN + 1; + char subpath[len]; + struct stat st; + while ( (entry = readdir( dir )) != NULL ) { + if ( strlen( entry->d_name ) > SUBDIR_LEN ) { + memlogf( "[WARNING] Skipping entry %s: Too long (max %d bytes)", entry->d_name, (int)SUBDIR_LEN ); + continue; + } + if ( entry->d_name[0] == '/' || path[pathLen - 1] == '/' ) { + snprintf( subpath, len, "%s%s", path, entry->d_name ); + } else { + snprintf( subpath, len, "%s/%s", path, entry->d_name ); + } + if ( stat( subpath, &st ) < 0 ) { + memlogf( "[WARNING] stat() for '%s' failed. Ignoring....", subpath ); + continue; + } + if ( S_ISDIR( st.st_mode )) { + image_load_all( base, subpath ); // Recurse + } else { + image_try_load( base, subpath ); // Load image if possible + } + } + closedir( dir ); + return TRUE; +#undef SUBDIR_LEN +} + +static int image_try_load(char *base, char *path) +{ + int i, revision; + assert( base != NULL ); + assert( path != NULL ); + assert( *path == '/' ); + assert( strncmp( path, base, strlen(base)) == 0 ); + assert( base[strlen(base) - 1] == '/' ); + char *lastSlash = strrchr( path, '/' ); + char *fileName = lastSlash + 1; + char imgName[strlen( path )]; + const int fileNameLen = strlen( fileName ); + char * const virtBase = path + strlen( base ); + // Copy virtual path + assert( *virtBase != '/' ); + char *src = virtBase, *dst = imgName; + while ( src < lastSlash ) { + *dst++ = *src++; + } + *dst = '\0'; + if ( _vmdkLegacyMode && strend( fileName, ".vmdk" ) ) { + // Easy - legacy mode, simply append full file name and set rid to 1 + strcat( dst, fileName ); + revision = 1; + } else { + // Try to parse *.r syntax + for (i = fileNameLen - 1; i > 1; --i) { + if ( fileName[i] < '0' || fileName[i] > '9' ) break; + } + if ( i == fileNameLen - 1 ) return FALSE; + if ( fileName[i] != 'r' ) return FALSE; + if ( fileName[i - 1] != '.' ) return FALSE; + revision = atoi( fileName + i + 1 ); + src = fileName; + while (src < fileName + i - 1) { + *dst++ = *src++; + } + *dst = '\0'; + } + if (revision <= 0) { + memlogf("[WARNING] Image '%s' has invalid revision ID %d", path, revision); + return FALSE; + } + // TODO: LOAD IMAGE DATA ETC. +} + +/* + void image_find_latest() + { + // Not in array or most recent rid is requested, try file system + if (revision != 0) { + // Easy case - specific RID + char + } else { + // Determine base directory where the image in question has to reside. + // Eg, the _basePath is "/srv/", requested image is "rz/ubuntu/default-13.04" + // Then searchPath has to be set to "/srv/rz/ubuntu" + char searchPath[strlen(_basePath) + len + 1]; + char *lastSlash = strrchr(name, '/'); + char *baseName; // Name of the image. In the example above, it will be "default-13.04" + if ( lastSlash == NULL ) { + *searchPath = '\0'; + baseName = name; + } else { + char *from = name, *to = searchPath; + while (from < lastSlash) *to++ = *from++; + *to = '\0'; + baseName = lastSlash + 1; + } + // Now we have the search path in our real file system and the expected image name. + // The revision naming sceme is .r, so if we're looking for revision 13, + // our example image has to be named default-13.04.r13 + } + } + */ diff --git a/src/server/image.h b/src/server/image.h index ea09940..6eb2b15 100644 --- a/src/server/image.h +++ b/src/server/image.h @@ -5,4 +5,14 @@ int image_is_complete(dnbd3_image_t *image); +int image_save_cache_map(dnbd3_image_t *image); + +dnbd3_image_t* image_get(char *name, uint16_t revision); + +void image_release(dnbd3_image_t *image); + +void image_load_all(); + + + #endif diff --git a/src/server/net.c b/src/server/net.c index e967d5c..f3438cd 100644 --- a/src/server/net.c +++ b/src/server/net.c @@ -31,7 +31,6 @@ #include "sockhelper.h" #include "helper.h" #include "server.h" -#include "saveload.h" #include "memlog.h" #include "../serialize.h" #include "../config.h" diff --git a/src/server/server.c b/src/server/server.c index b2e5ba7..87b7c08 100644 --- a/src/server/server.c +++ b/src/server/server.c @@ -58,6 +58,8 @@ char *_config_file_name = DEFAULT_SERVER_CONFIG_FILE; char *_rpc_password = NULL; char *_cache_dir = NULL; +static void dnbd3_handle_sigpipe(int signum); +static void dnbd3_handle_sigterm(int signum); void dnbd3_print_help(char *argv_0) { @@ -85,7 +87,6 @@ void dnbd3_print_version() void dnbd3_cleanup() { int fd, i; - GSList *iterator = NULL; memlogf("INFO: Cleanup...\n"); @@ -98,12 +99,13 @@ void dnbd3_cleanup() } socket_count = 0; + // Clean up clients pthread_spin_lock(&_clients_lock); for (i = 0; i < _num_clients; ++i) { dnbd3_client_t * const client = _clients[i]; pthread_spin_lock(&client->lock); - if (client->sock != -1) shutdown(client->sock, SHUT_RDWR); + if (client->sock >= 0) shutdown(client->sock, SHUT_RDWR); if (client->thread != 0) pthread_join(client->thread, NULL); _clients[i] = NULL; pthread_spin_unlock(&client->lock); @@ -112,41 +114,25 @@ void dnbd3_cleanup() _num_clients = 0; pthread_spin_unlock(&_clients_lock); + // Clean up images pthread_spin_lock(&_images_lock); for (i = 0; i < _num_images; ++i) { - // save cache maps to files dnbd3_image_t *image = _images[i]; - if (image->cache_file) - { - char tmp[strlen(image->cache_file)+4]; - strcpy(tmp, image->cache_file); - strcat(tmp, ".map"); - fd = open(tmp, O_WRONLY | O_CREAT, S_IRUSR | S_IWUSR); - - if (fd > 0) - write(fd, image->cache_map, ((image->filesize + (1 << 15) - 1) >> 15) * sizeof(char)); - - close(fd); - } - // Close bock devices of proxied images - if (image->file && strncmp(image->file, "/dev/dnbd", 9) == 0) - { - int fd = open(image->file, O_RDONLY); - dnbd3_ioctl_t msg; - memset(&msg, 0, sizeof(msg)); - msg.len = sizeof(msg); - ioctl(fd, IOCTL_CLOSE, &msg); - close(fd); - } - + pthread_spin_lock(&image->lock); + // save cache maps to files + image_save_cache_map(image); + // free uplink connection + uplink_free(image->uplink); + // free other stuff free(image->cache_map); - free(image->config_group); - free(image->low_name); - free(image->file); - free(image->cache_file); - g_free(image); + free(image->path); + free(image->lower_name); + _images[i] = NULL; + pthread_spin_unlock(&image->lock); + free(image); } + _num_images = 0; pthread_spin_unlock(&_images_lock); exit(EXIT_SUCCESS); @@ -157,6 +143,7 @@ int main(int argc, char *argv[]) int demonize = 1; int opt = 0; int longIndex = 0; + int i; static const char *optString = "f:d:nrsiHV?"; static const struct option longOpts[] = { @@ -218,7 +205,9 @@ int main(int argc, char *argv[]) if (demonize) daemon(1, 0); - pthread_spin_init(&_spinlock, PTHREAD_PROCESS_PRIVATE); + pthread_spin_init(&_clients_lock, PTHREAD_PROCESS_PRIVATE); + pthread_spin_init(&_images_lock, PTHREAD_PROCESS_PRIVATE); + pthread_spin_init(&_alts_lock, PTHREAD_PROCESS_PRIVATE); initmemlog(); memlogf("DNBD3 server starting.... Machine type: " ENDIAN_MODE); @@ -247,12 +236,12 @@ int main(int argc, char *argv[]) int fd; // setup rpc - pthread_t thread_rpc; - pthread_create(&(thread_rpc), NULL, &dnbd3_rpc_mainloop, NULL); + //pthread_t thread_rpc; + //pthread_create(&(thread_rpc), NULL, &dnbd3_rpc_mainloop, NULL); // setup the job thread (query other servers, delete old images etc.) - pthread_t thread_job; - pthread_create(&(thread_job), NULL, &dnbd3_job_thread, NULL); + //pthread_t thread_job; + //pthread_create(&(thread_job), NULL, &dnbd3_job_thread, NULL); memlogf("[INFO] Server is ready..."); @@ -264,55 +253,31 @@ int main(int argc, char *argv[]) if (fd < 0) { memlogf("[ERROR] Client accept failure"); + usleep(10000); // 10ms continue; } //memlogf("INFO: Client %s connected\n", inet_ntoa(client.sin_addr)); sock_set_timeout(fd, SOCKET_TIMEOUT_SERVER_MS); - dnbd3_client_t *dnbd3_client = g_new0(dnbd3_client_t, 1); + dnbd3_client_t *dnbd3_client = dnbd3_init_client(&client, fd); if (dnbd3_client == NULL) { - memlogf("[ERROR] Could not alloc dnbd3_client_t for new client."); close(fd); continue; } - if (client.ss_family == AF_INET) { - struct sockaddr_in *v4 = (struct sockaddr_in *)&client; - dnbd3_client->host.type = AF_INET; - memcpy(dnbd3_client->host.addr, &(v4->sin_addr), 4); - dnbd3_client->host.port = v4->sin_port; - } - else if (client.ss_family == AF_INET6) - { - struct sockaddr_in6 *v6 = (struct sockaddr_in6 *)&client; - dnbd3_client->host.type = AF_INET6; - memcpy(dnbd3_client->host.addr, &(v6->sin6_addr), 16); - dnbd3_client->host.port = v6->sin6_port; - } - else - { - memlogf("[ERROR] New client has unknown address family %d, disconnecting...", (int)client.ss_family); - close(fd); - g_free(dnbd3_client); - continue; - } - dnbd3_client->sock = fd; - dnbd3_client->image = NULL; // This has to be done before creating the thread, otherwise a race condition might occur when the new thread dies faster than this thread adds the client to the list after creating the thread - pthread_spin_lock(&_spinlock); - _dnbd3_clients = g_slist_prepend(_dnbd3_clients, dnbd3_client); - pthread_spin_unlock(&_spinlock); + if (!dnbd3_add_client(dnbd3_client)) { + dnbd3_free_client(dnbd3_client); + continue; + } if (0 != pthread_create(&(dnbd3_client->thread), NULL, net_client_handler, (void *) (uintptr_t) dnbd3_client)) { memlogf("[ERROR] Could not start thread for new client."); - pthread_spin_lock(&_spinlock); - _dnbd3_clients = g_slist_remove(_dnbd3_clients, dnbd3_client); - pthread_spin_unlock(&_spinlock); + dnbd3_remove_client(dnbd3_client); dnbd3_free_client(dnbd3_client); - close(fd); continue; } pthread_detach(dnbd3_client->thread); @@ -321,6 +286,34 @@ int main(int argc, char *argv[]) dnbd3_cleanup(); } +dnbd3_client_t* dnbd3_init_client(struct sockaddr_storage *client, int fd) +{ + dnbd3_client_t *dnbd3_client = calloc(1, sizeof(dnbd3_client_t)); + if (dnbd3_client == NULL) { + memlogf("[ERROR] Could not alloc dnbd3_client_t for new client."); + return NULL; + } + + if (client.ss_family == AF_INET) { + struct sockaddr_in *v4 = (struct sockaddr_in *)&client; + dnbd3_client->host.type = AF_INET; + memcpy(dnbd3_client->host.addr, &(v4->sin_addr), 4); + dnbd3_client->host.port = v4->sin_port; + } else if (client.ss_family == AF_INET6) { + struct sockaddr_in6 *v6 = (struct sockaddr_in6 *)&client; + dnbd3_client->host.type = AF_INET6; + memcpy(dnbd3_client->host.addr, &(v6->sin6_addr), 16); + dnbd3_client->host.port = v6->sin6_port; + } else { + memlogf("[ERROR] New client has unknown address family %d, disconnecting...", (int)client.ss_family); + free(dnbd3_client); + return NULL; + } + dnbd3_client->sock = fd; + pthread_spin_init(&dnbd3_client->lock, PTHREAD_PROCESS_PRIVATE); + return dnbd3_client; +} + /** * Free the client struct recursively */ @@ -332,5 +325,49 @@ void dnbd3_free_client(dnbd3_client_t *client) free(it->data); } g_slist_free(client->sendqueue); + if (client->sock >= 0) close(client->sock); g_free(client); } + +int dnbd3_add_client(dnbd3_client_t *client) +{ + int i; + pthread_spin_lock(&_clients_lock); + for (i = 0; i < _num_clients; ++i) { + if (_clients[i] != NULL) continue; + _clients[i] = client; + pthread_spin_unlock(&_clients_lock); + return TRUE; + } + if (_num_clients >= SERVER_MAX_CLIENTS) { + pthread_spin_unlock(&_clients_lock); + memlogf("[ERROR] Maximum number of clients reached!"); + return FALSE; + } + _clients[_num_clients++] = client; + pthread_spin_unlock(&_clients_lock); + return TRUE; +} + +void dnbd3_remove_client(dnbd3_client_t *client) +{ + int i; + pthread_spin_lock(&_clients_lock); + for (i = _num_clients - 1; i >= 0; --i) { + if (_clients[i] != client) continue; + _clients[i] = NULL; + if (i + 1 == _num_clients) --_num_clients; + } + pthread_spin_unlock(&_clients_lock); +} + +static void dnbd3_handle_sigpipe(int signum) +{ + memlogf("INFO: SIGPIPE received (%s)", strsignal(signum)); +} + +static void dnbd3_handle_sigterm(int signum) +{ + memlogf("INFO: SIGTERM or SIGINT received (%s)", strsignal(signum)); + dnbd3_cleanup(); +} diff --git a/src/server/server.h b/src/server/server.h index 74ca1a1..410cc60 100644 --- a/src/server/server.h +++ b/src/server/server.h @@ -43,13 +43,14 @@ typedef struct { char *path; // absolute path of the image char *lower_name; // relative path, all lowercase, minus revision ID - int rid; // revision of image - uint64_t filesize; // size of image - time_t atime; // last access time uint8_t *cache_map; // cache map telling which parts are locally cached dnbd3_connection_t *uplink; // NULL = local image / completely cached, pointer to a server connection otherwise + uint64_t filesize; // size of image + int rid; // revision of image + int users; // clients currently using this image + time_t atime; // last access time char working; // TRUE if image exists and completeness is == 100% or a working upstream proxy is connected - time_t delete_soft; // unixtime telling when this image should be deleted. if there are still clients using this image it weill be kept, but new clients requesting the image will be rejected. 0 = never + time_t delete_soft; // unixtime telling when this image should be deleted. if there are still clients using this image it weill be kept, but new clients requesting the image will be rejected. 0 = never time_t delete_hard; // unixtime telling when this image should be deleted, no matter if there are still clients connected. 0 = never pthread_spinlock_t lock; } dnbd3_image_t; @@ -78,16 +79,16 @@ typedef struct typedef struct { - time_t last_told; - dnbd3_host_t host; - char comment[COMMENT_LENGTH]; + time_t last_told; + dnbd3_host_t host; + char comment[COMMENT_LENGTH]; } dnbd3_alt_server_t; typedef struct { - char comment[COMMENT_LENGTH]; - dnbd3_host_t host; - dnbd3_host_t mask; + char comment[COMMENT_LENGTH]; + dnbd3_host_t host; + dnbd3_host_t mask; } dnbd3_acess_rules_t; extern dnbd3_client_t *_clients[SERVER_MAX_CLIENTS]; @@ -109,6 +110,9 @@ extern int _fake_delay; #endif void dnbd3_cleanup(); +void dnbd3_add_client(dnbd3_client_t *client); +void dnbd3_remove_client(dnbd3_client_t *client); +dnbd3_client_t* dnbd3_init_client(struct sockaddr_storage *client, int fd); void dnbd3_free_client(dnbd3_client_t *client); #if !defined(_FILE_OFFSET_BITS) || _FILE_OFFSET_BITS != 64 -- cgit v1.2.3-55-g7522