From be7d7d95850c30a154aaa56e95d6a7f36793409d Mon Sep 17 00:00:00 2001 From: Simon Rettberg Date: Wed, 7 Aug 2019 17:11:51 +0200 Subject: [SERVER] Better lock debugging: Always check lock order Lock order is predefined in locks.h. Immediately bail out if a lock with lower priority is obtained while the same thread already holds one with higher priority. --- LOCKS | 13 +- src/server/altservers.c | 9 +- src/server/globals.c | 2 +- src/server/image.c | 10 +- src/server/integrity.c | 2 +- src/server/locks.c | 319 ++++++++++++++++++++++-------------------------- src/server/locks.h | 36 ++++-- src/server/net.c | 6 +- src/server/rpc.c | 14 +-- src/server/server.c | 7 -- src/server/uplink.c | 6 +- 11 files changed, 198 insertions(+), 226 deletions(-) diff --git a/LOCKS b/LOCKS index 4b5b07c..77e44a8 100644 --- a/LOCKS +++ b/LOCKS @@ -16,23 +16,22 @@ requests.lock ===== SERVER ===== This is a list of used locks, in the order they -have to be aquired if you must hold multiple locks: -remoteCloneLock | reloadLock +have to be aquired if you must hold multiple locks. +Note this list might be out of date, take a look at the +defines in lock.h for the effective order. +reloadLock +remoteCloneLock _clients_lock _clients[].lock integrityQueueLock _images_lock _images[].lock -pendingLockConsume -pendingLockProduce uplink.queueLock altServersLock client.sendMutex -client.statsLock -statisticsSentLock -statisticsReceivedLock uplink.rttLock uplink.sendMutex +aclLock If you need to lock multiple clients/images/... at once, lock the client with the lowest array index first. diff --git a/src/server/altservers.c b/src/server/altservers.c index a270bf3..3d5e71e 100644 --- a/src/server/altservers.c +++ b/src/server/altservers.c @@ -30,7 +30,7 @@ void altservers_init() { srand( (unsigned int)time( NULL ) ); // Init spinlock - mutex_init( &altServersLock ); + mutex_init( &altServersLock, LOCK_ALT_SERVER_LIST ); // Init signal runSignal = signal_new(); if ( runSignal == NULL ) { @@ -326,13 +326,13 @@ json_t* altservers_toJson() } /** - * Update rtt history of given server - returns the new average for that server + * Update rtt history of given server - returns the new average for that server. + * XXX HOLD altServersLock WHEN CALLING THIS! */ static unsigned int altservers_updateRtt(const dnbd3_host_t * const host, const unsigned int rtt) { unsigned int avg = rtt; int i; - mutex_lock( &altServersLock ); for (i = 0; i < numAltServers; ++i) { if ( !isSameAddressPort( host, &altServers[i].host ) ) continue; altServers[i].rtt[++altServers[i].rttIndex % SERVER_RTT_PROBES] = rtt; @@ -353,7 +353,6 @@ static unsigned int altservers_updateRtt(const dnbd3_host_t * const host, const } break; } - mutex_unlock( &altServersLock ); return avg; } @@ -529,6 +528,7 @@ static void *altservers_main(void *data UNUSED) } clock_gettime( BEST_CLOCK_SOURCE, &end ); // Measurement done - everything fine so far + mutex_lock( &altServersLock ); mutex_lock( &uplink->rttLock ); const bool isCurrent = isSameAddressPort( &servers[itAlt], &uplink->currentServer ); // Penaltize rtt if this was a cycle; this will treat this server with lower priority @@ -538,6 +538,7 @@ static void *altservers_main(void *data UNUSED) + (end.tv_nsec - start.tv_nsec) / 1000 + ( (isCurrent && uplink->cycleDetected) ? 1000000 : 0 )); // µs unsigned int avg = altservers_updateRtt( &servers[itAlt], rtt ); + mutex_unlock( &altServersLock ); // If a cycle was detected, or we lost connection to the current (last) server, penaltize it one time if ( ( uplink->cycleDetected || uplink->fd == -1 ) && isCurrent ) avg = (avg * 2) + 50000; mutex_unlock( &uplink->rttLock ); diff --git a/src/server/globals.c b/src/server/globals.c index 69e8a6e..46c1030 100644 --- a/src/server/globals.c +++ b/src/server/globals.c @@ -112,7 +112,7 @@ void globals_loadConfig() asprintf( &name, "%s/%s", _configDir, CONFIG_FILENAME ); if ( name == NULL ) return; if ( initialLoad ) { - mutex_init( &loadLock ); + mutex_init( &loadLock, LOCK_LOAD_CONFIG ); } if ( mutex_trylock( &loadLock ) != 0 ) { logadd( LOG_INFO, "Ignoring config reload request due to already running reload" ); diff --git a/src/server/image.c b/src/server/image.c index 1f12eda..4a65ed3 100644 --- a/src/server/image.c +++ b/src/server/image.c @@ -59,9 +59,9 @@ static bool image_checkRandomBlocks(const int count, int fdImage, const int64_t void image_serverStartup() { srand( (unsigned int)time( NULL ) ); - mutex_init( &imageListLock ); - mutex_init( &remoteCloneLock ); - mutex_init( &reloadLock ); + mutex_init( &imageListLock, LOCK_IMAGE_LIST ); + mutex_init( &remoteCloneLock, LOCK_REMOTE_CLONE ); + mutex_init( &reloadLock, LOCK_RELOAD ); } /** @@ -347,7 +347,7 @@ dnbd3_image_t* image_get(char *name, uint16_t revision, bool checkIfWorking) img->rid = candidate->rid; img->users = 1; img->working = false; - mutex_init( &img->lock ); + mutex_init( &img->lock, LOCK_IMAGE ); if ( candidate->crc32 != NULL ) { const size_t mb = IMGSIZE_TO_HASHBLOCKS( candidate->virtualFilesize ) * sizeof(uint32_t); img->crc32 = malloc( mb ); @@ -869,7 +869,7 @@ static bool image_load(char *base, char *path, int withUplink) image->working = (image->cache_map == NULL ); timing_get( &image->nextCompletenessEstimate ); image->completenessEstimate = -1; - mutex_init( &image->lock ); + mutex_init( &image->lock, LOCK_IMAGE ); int32_t offset; if ( stat( path, &st ) == 0 ) { // Negatively offset atime by file modification time diff --git a/src/server/integrity.c b/src/server/integrity.c index a66a364..c52d17b 100644 --- a/src/server/integrity.c +++ b/src/server/integrity.c @@ -39,7 +39,7 @@ static void* integrity_main(void *data); void integrity_init() { assert( queueLen == -1 ); - mutex_init( &integrityQueueLock ); + mutex_init( &integrityQueueLock, LOCK_INTEGRITY_QUEUE ); pthread_cond_init( &queueSignal, NULL ); mutex_lock( &integrityQueueLock ); queueLen = 0; diff --git a/src/server/locks.c b/src/server/locks.c index 2c0cb27..b39576b 100644 --- a/src/server/locks.c +++ b/src/server/locks.c @@ -12,47 +12,45 @@ #ifdef _DEBUG #define MAXLOCKS (SERVER_MAX_CLIENTS * 2 + SERVER_MAX_ALTS + 200 + SERVER_MAX_IMAGES) #define MAXTHREADS (SERVER_MAX_CLIENTS + 100) +#define MAXLPT 20 #define LOCKLEN 60 typedef struct { - void *lock; + void * _Atomic lock; ticks locktime; - char locked; - pthread_t thread; + bool _Atomic locked; + pthread_t _Atomic thread; int lockId; + int prio; char name[LOCKLEN]; char where[LOCKLEN]; } debug_lock_t; typedef struct { - pthread_t tid; + pthread_t _Atomic tid; ticks time; char name[LOCKLEN]; char where[LOCKLEN]; - + debug_lock_t *locks[MAXLPT]; } debug_thread_t; int debugThreadCount = 0; static debug_lock_t locks[MAXLOCKS]; static debug_thread_t threads[MAXTHREADS]; -static int init_done = 0; -static pthread_mutex_t initdestory; +static pthread_mutex_t initdestory = PTHREAD_MUTEX_INITIALIZER; static int lockId = 0; -static pthread_t watchdog = 0; -static dnbd3_signal_t* watchdogSignal = NULL; -static void *debug_thread_watchdog(void *something); +#define ULDE(...) do { \ + pthread_mutex_unlock( &initdestory ); \ + logadd( LOG_ERROR, __VA_ARGS__ ); \ + debug_dump_lock_stats(); \ + exit( 4 ); \ +} while(0) -int debug_mutex_init(const char *name, const char *file, int line, pthread_mutex_t *lock) +int debug_mutex_init(const char *name, const char *file, int line, pthread_mutex_t *lock, int priority) { - if ( !init_done ) { - memset( locks, 0, MAXLOCKS * sizeof(debug_lock_t) ); - memset( threads, 0, MAXTHREADS * sizeof(debug_thread_t) ); - pthread_mutex_init( &initdestory, NULL ); - init_done = 1; - } int first = -1; pthread_mutex_lock( &initdestory ); for (int i = 0; i < MAXLOCKS; ++i) { @@ -63,20 +61,18 @@ int debug_mutex_init(const char *name, const char *file, int line, pthread_mutex if ( first == -1 && locks[i].lock == NULL ) first = i; } if ( first == -1 ) { - logadd( LOG_ERROR, "No more free debug locks (%s:%d)\n", file, line ); - pthread_mutex_unlock( &initdestory ); - debug_dump_lock_stats(); - exit( 4 ); + ULDE( "No more free debug locks (%s:%d)\n", file, line ); } locks[first].lock = (void*)lock; - locks[first].locked = 0; + locks[first].locked = false; + locks[first].prio = priority; snprintf( locks[first].name, LOCKLEN, "%s", name ); snprintf( locks[first].where, LOCKLEN, "I %s:%d", file, line ); pthread_mutex_unlock( &initdestory ); return pthread_mutex_init( lock, NULL ); } -int debug_mutex_lock(const char *name, const char *file, int line, pthread_mutex_t *lock) +int debug_mutex_lock(const char *name, const char *file, int line, pthread_mutex_t *lock, bool try) { debug_lock_t *l = NULL; pthread_mutex_lock( &initdestory ); @@ -86,163 +82,180 @@ int debug_mutex_lock(const char *name, const char *file, int line, pthread_mutex break; } } - pthread_mutex_unlock( &initdestory ); if ( l == NULL ) { - logadd( LOG_ERROR, "Tried to lock uninitialized lock %p (%s) at %s:%d\n", (void*)lock, name, file, line ); - debug_dump_lock_stats(); - exit( 4 ); + ULDE( "Tried to lock uninitialized lock %p (%s) at %s:%d\n", (void*)lock, name, file, line ); } debug_thread_t *t = NULL; - pthread_mutex_lock( &initdestory ); + int first = -1; + const pthread_t self = pthread_self(); for (int i = 0; i < MAXTHREADS; ++i) { - if ( threads[i].tid != 0 ) continue; - threads[i].tid = pthread_self(); - timing_get( &threads[i].time ); - snprintf( threads[i].name, LOCKLEN, "%s", name ); - snprintf( threads[i].where, LOCKLEN, "%s:%d", file, line ); - t = &threads[i]; - break; - } - pthread_mutex_unlock( &initdestory ); - if ( t == NULL ) { - logadd( LOG_ERROR, "Lock sanity check: Too many waiting threads for lock %p (%s) at %s:%d\n", (void*)lock, name, file, line ); - exit( 4 ); - } - const int retval = pthread_mutex_lock( lock ); - pthread_mutex_lock( &initdestory ); - t->tid = 0; - pthread_mutex_unlock( &initdestory ); - if ( l->locked ) { - logadd( LOG_ERROR, "Lock sanity check: lock %p (%s) already locked at %s:%d\n", (void*)lock, name, file, line ); - exit( 4 ); - } - l->locked = 1; - timing_get( &l->locktime ); - l->thread = pthread_self(); - snprintf( l->where, LOCKLEN, "L %s:%d", file, line ); - pthread_mutex_lock( &initdestory ); - l->lockId = ++lockId; - pthread_mutex_unlock( &initdestory ); - return retval; -} - -int debug_mutex_trylock(const char *name, const char *file, int line, pthread_mutex_t *lock) -{ - debug_lock_t *l = NULL; - pthread_mutex_lock( &initdestory ); - for (int i = 0; i < MAXLOCKS; ++i) { - if ( locks[i].lock == lock ) { - l = &locks[i]; + if ( threads[i].tid == self ) { + t = &threads[i]; break; } + if ( first == -1 && threads[i].tid == 0 ) { + first = i; + } } - pthread_mutex_unlock( &initdestory ); - if ( l == NULL ) { - logadd( LOG_ERROR, "Tried to lock uninitialized lock %p (%s) at %s:%d\n", (void*)lock, name, file, line ); - debug_dump_lock_stats(); - exit( 4 ); - } - debug_thread_t *t = NULL; - pthread_mutex_lock( &initdestory ); - for (int i = 0; i < MAXTHREADS; ++i) { - if ( threads[i].tid != 0 ) continue; - threads[i].tid = pthread_self(); - timing_get( &threads[i].time ); - snprintf( threads[i].name, LOCKLEN, "%s", name ); - snprintf( threads[i].where, LOCKLEN, "%s:%d", file, line ); - t = &threads[i]; - break; - } - pthread_mutex_unlock( &initdestory ); + int idx; if ( t == NULL ) { - logadd( LOG_ERROR, "Lock sanity check: Too many waiting threads for %p (%s) at %s:%d\n", (void*)lock, name, file, line ); - exit( 4 ); + if ( first == -1 ) { + ULDE( "Lock sanity check: Too many waiting threads for lock %p (%s) at %s:%d\n", (void*)lock, name, file, line ); + } + t = &threads[first]; + timing_get( &t->time ); + t->tid = self; + snprintf( t->name, LOCKLEN, "%s", name ); + snprintf( t->where, LOCKLEN, "%s:%d", file, line ); + memset( t->locks, 0, sizeof(t->locks) ); + idx = 0; + } else { + // Thread already has locks, check for order violation + idx = -1; + for (int i = 0; i < MAXLPT; ++i) { + if ( t->locks[i] == NULL ) { + if ( idx == -1 ) { + idx = i; + } + continue; + } + if ( t->locks[i]->prio >= l->prio ) { + ULDE( "Lock priority violation: %s at %s:%d (%d) when already holding %s at %s (%d)", + name, file, line, l->prio, + t->locks[i]->name, t->locks[i]->where, t->locks[i]->prio ); + } + if ( t->locks[i] == l ) { + ULDE( "Tried to recusively lock %s in the same thread. Tried at %s:%d, when already locked at %s", + name, file, line, t->locks[i]->name ); + } + } + if ( idx == -1 ) { + ULDE( "Thread %d tried to lock more than %d locks.", (int)self, (int)MAXLPT ); + } } - const int retval = pthread_mutex_trylock( lock ); - pthread_mutex_lock( &initdestory ); - t->tid = 0; pthread_mutex_unlock( &initdestory ); + const int retval = try ? pthread_mutex_trylock( lock ) : pthread_mutex_lock( lock ); if ( retval == 0 ) { + timing_get( &l->locktime ); + l->thread = self; + snprintf( l->where, LOCKLEN, "L %s:%d", file, line ); + pthread_mutex_lock( &initdestory ); if ( l->locked ) { logadd( LOG_ERROR, "Lock sanity check: lock %p (%s) already locked at %s:%d\n", (void*)lock, name, file, line ); exit( 4 ); } - l->locked = 1; - timing_get( &l->locktime ); - l->thread = pthread_self(); - snprintf( l->where, LOCKLEN, "L %s:%d", file, line ); - pthread_mutex_lock( &initdestory ); + l->locked = true; + t->locks[idx] = l; l->lockId = ++lockId; pthread_mutex_unlock( &initdestory ); + } else if ( !try || retval != EBUSY ) { + logadd( LOG_ERROR, "Acquiring lock %s at %s:%d failed with error code %d", name, file, line, retval ); + debug_dump_lock_stats(); + exit( 4 ); } return retval; } int debug_mutex_unlock(const char *name, const char *file, int line, pthread_mutex_t *lock) { - debug_lock_t *l = NULL; + debug_thread_t *t = NULL; + pthread_t self = pthread_self(); pthread_mutex_lock( &initdestory ); - for (int i = 0; i < MAXLOCKS; ++i) { - if ( locks[i].lock == lock ) { - l = &locks[i]; + for (int i = 0; i < MAXTHREADS; ++i) { + if ( threads[i].tid == self ) { + t = &threads[i]; break; } } - pthread_mutex_unlock( &initdestory ); - if ( l == NULL ) { - logadd( LOG_ERROR, "Tried to unlock uninitialized lock %p (%s) at %s:%d\n", (void*)lock, name, file, line ); - exit( 4 ); + if ( t == NULL ) { + ULDE( "Unlock called from unknown thread for %s at %s:%d", name, file, line ); } - if ( !l->locked ) { - logadd( LOG_ERROR, "Unlock sanity check: lock %p (%s) not locked at %s:%d\n", (void*)lock, name, file, line ); - exit( 4 ); + int idx = -1; + int cnt = 0; + for (int i = 0; i < MAXLPT; ++i) { + if ( t->locks[i] == NULL ) + continue; + cnt++; + if ( t->locks[i]->lock == lock ) { + idx = i; + } + } + if ( idx == -1 ) { + ULDE( "Unlock: Calling thread doesn't hold lock %s at %s:%d", name, file, line ); } - l->locked = 0; + debug_lock_t *l = t->locks[idx]; + if ( l->thread != self || !l->locked ) { + ULDE( "Unlock sanity check for lock debugger failed! Lock %s is assigned to calling thread, but lock's meta data doesn't match up at %s:%d", name, file, line ); + } + l->locked = false; l->thread = 0; + t->locks[idx] = NULL; + if ( cnt == 1 ) { + t->tid = 0; // No more locks held, free up slot + } snprintf( l->where, LOCKLEN, "U %s:%d", file, line ); - int retval = pthread_mutex_unlock( lock ); + pthread_mutex_unlock( &initdestory ); + const int retval = pthread_mutex_unlock( lock ); + if ( retval != 0 ) { + logadd( LOG_ERROR, "pthread_mutex_unlock returned %d for %s at %s:%d", retval, name, file, line ); + exit( 4 ); + } return retval; } int debug_mutex_cond_wait(const char *name, const char *file, int line, pthread_cond_t *restrict cond, pthread_mutex_t *restrict lock) { debug_lock_t *l = NULL; + debug_thread_t *t = NULL; + pthread_t self = pthread_self(); pthread_mutex_lock( &initdestory ); - for (int i = 0; i < MAXLOCKS; ++i) { - if ( locks[i].lock == lock ) { - l = &locks[i]; + for (int i = 0; i < MAXTHREADS; ++i) { + if ( threads[i].tid == self ) { + t = &threads[i]; break; } } - pthread_mutex_unlock( &initdestory ); + if ( t == NULL ) { + ULDE( "Unlock called from unknown thread for %s at %s:%d", name, file, line ); + } + int mp = 0, mpi = -1; + for (int i = 0; i < MAXLPT; ++i) { + if ( t->locks[i] == NULL ) + continue; + if ( t->locks[i]->lock == lock ) { + l = t->locks[i]; + } else if ( t->locks[i]->prio > mp ) { + mp = t->locks[i]->prio; + mpi = i; + } + } if ( l == NULL ) { - logadd( LOG_ERROR, "Tried to cond_wait on uninitialized lock %p (%s) at %s:%d\n", (void*)lock, name, file, line ); - exit( 4 ); + ULDE( "cond_wait: Calling thread doesn't hold lock %s at %s:%d", name, file, line ); } - if ( !l->locked ) { - logadd( LOG_ERROR, "Cond_wait sanity check: lock %p (%s) not locked at %s:%d\n", (void*)lock, name, file, line ); - exit( 4 ); + if ( l->thread != self || !l->locked ) { + ULDE( "cond_wait: Sanity check for lock debugger failed! Lock %s is assigned to calling thread, but lock's meta data doesn't match up at %s:%d", name, file, line ); } - pthread_t self = pthread_self(); - if ( l->thread != self ) { - logadd( LOG_ERROR, "Cond_wait called from non-owning thread for %p (%s) at %s:%d\n", (void*)lock, name, file, line ); - exit( 4 ); + if ( mp >= l->prio ) { + ULDE( "cond_wait: Yielding a mutex while holding another one with higher prio: %s at %s:%d (%d) while also holding %s at %s (%d)", + name, file, line, l->prio, + t->locks[mpi]->name, t->locks[mpi]->where, mp ); } - l->locked = 0; + l->locked = false; l->thread = 0; - snprintf( l->where, LOCKLEN, "CW %s:%d", file, line ); + snprintf( l->where, LOCKLEN, "CWU %s:%d", file, line ); + pthread_mutex_unlock( &initdestory ); int retval = pthread_cond_wait( cond, lock ); if ( retval != 0 ) { logadd( LOG_ERROR, "pthread_cond_wait returned %d for lock %p (%s) at %s:%d\n", retval, (void*)lock, name, file, line ); exit( 4 ); } - if ( l->locked != 0 || l->thread != 0 ) { + if ( l->locked || l->thread != 0 ) { logadd( LOG_ERROR, "Lock is not free after returning from pthread_cond_wait for %p (%s) at %s:%d\n", (void*)lock, name, file, line ); exit( 4 ); } - l->locked = 1; l->thread = self; timing_get( &l->locktime ); + l->locked = true; pthread_mutex_lock( &initdestory ); l->lockId = ++lockId; pthread_mutex_unlock( &initdestory ); @@ -290,63 +303,21 @@ void debug_dump_lock_stats() "* Locked: %d\n", locks[i].name, locks[i].where, (int)locks[i].locked ); } } - printf( "\n **** WAITING THREADS ****\n\n" ); + printf( "\n **** ACTIVE THREADS ****\n\n" ); for (int i = 0; i < MAXTHREADS; ++i) { - if ( threads[i].tid == 0 ) continue; + if ( threads[i].tid == 0 ) + continue; printf( "* *** Thread %d ***\n" "* Lock: %s\n" "* Where: %s\n" "* How long: %d secs\n", (int)threads[i].tid, threads[i].name, threads[i].where, (int)timing_diff( &threads[i].time, &now ) ); - } - pthread_mutex_unlock( &initdestory ); -} - -static void *debug_thread_watchdog(void *something UNUSED) -{ - setThreadName( "debug-watchdog" ); - while ( !_shutdown ) { - if ( init_done ) { - declare_now; - pthread_mutex_lock( &initdestory ); - for (int i = 0; i < MAXTHREADS; ++i) { - if ( threads[i].tid == 0 ) continue; - const uint32_t diff = timing_diff( &threads[i].time, &now ); - if ( diff > 6 && diff < 100000 ) { - printf( "\n\n +++++++++ DEADLOCK ++++++++++++\n\n" ); - pthread_mutex_unlock( &initdestory ); - debug_dump_lock_stats(); - exit( 99 ); - } - } - pthread_mutex_unlock( &initdestory ); + for (int j = 0; j < MAXLPT; ++j) { + if ( threads[i].locks[j] == NULL ) + continue; + printf( " * Lock %s @ %s\n", threads[i].locks[j]->name, threads[i].locks[j]->where ); } - if ( watchdogSignal == NULL || signal_wait( watchdogSignal, 5000 ) == SIGNAL_ERROR ) sleep( 5 ); } - return NULL ; -} - -#endif - -void debug_locks_start_watchdog() -{ -#ifdef _DEBUG - watchdogSignal = signal_new(); - if ( 0 != thread_create( &watchdog, NULL, &debug_thread_watchdog, (void *)NULL ) ) { - logadd( LOG_ERROR, "Could not start debug-lock watchdog." ); - return; - } -#endif + pthread_mutex_unlock( &initdestory ); } -void debug_locks_stop_watchdog() -{ -#ifdef _DEBUG - _shutdown = true; - printf( "Killing debug watchdog...\n" ); - pthread_mutex_lock( &initdestory ); - signal_call( watchdogSignal ); - pthread_mutex_unlock( &initdestory ); - thread_join( watchdog, NULL ); - signal_close( watchdogSignal ); #endif -} diff --git a/src/server/locks.h b/src/server/locks.h index 7f72722..e5c9801 100644 --- a/src/server/locks.h +++ b/src/server/locks.h @@ -5,19 +5,38 @@ #include #include #include +#include + +// Lock priority + +#define LOCK_RELOAD 90 +#define LOCK_LOAD_CONFIG 100 +#define LOCK_REMOTE_CLONE 110 +#define LOCK_CLIENT_LIST 120 +#define LOCK_CLIENT 130 +#define LOCK_INTEGRITY_QUEUE 140 +#define LOCK_IMAGE_LIST 150 +#define LOCK_IMAGE 160 +#define LOCK_UPLINK_QUEUE 170 +#define LOCK_ALT_SERVER_LIST 180 +#define LOCK_CLIENT_SEND 190 +#define LOCK_UPLINK_RTT 200 +#define LOCK_UPLINK_SEND 210 +#define LOCK_RPC_ACL 220 + +// #ifdef _DEBUG -#define mutex_init( lock ) debug_mutex_init( #lock, __FILE__, __LINE__, lock) -#define mutex_lock( lock ) debug_mutex_lock( #lock, __FILE__, __LINE__, lock) -#define mutex_trylock( lock ) debug_mutex_trylock( #lock, __FILE__, __LINE__, lock) +#define mutex_init( lock, prio ) debug_mutex_init( #lock, __FILE__, __LINE__, lock, prio) +#define mutex_lock( lock ) debug_mutex_lock( #lock, __FILE__, __LINE__, lock, false) +#define mutex_trylock( lock ) debug_mutex_lock( #lock, __FILE__, __LINE__, lock, true) #define mutex_unlock( lock ) debug_mutex_unlock( #lock, __FILE__, __LINE__, lock) #define mutex_cond_wait( cond, lock ) debug_mutex_cond_wait( #lock, __FILE__, __LINE__, cond, lock) #define mutex_destroy( lock ) debug_mutex_destroy( #lock, __FILE__, __LINE__, lock) -int debug_mutex_init(const char *name, const char *file, int line, pthread_mutex_t *lock); -int debug_mutex_lock(const char *name, const char *file, int line, pthread_mutex_t *lock); -int debug_mutex_trylock(const char *name, const char *file, int line, pthread_mutex_t *lock); +int debug_mutex_init(const char *name, const char *file, int line, pthread_mutex_t *lock, int priority); +int debug_mutex_lock(const char *name, const char *file, int line, pthread_mutex_t *lock, bool try); int debug_mutex_unlock(const char *name, const char *file, int line, pthread_mutex_t *lock); int debug_mutex_cond_wait(const char *name, const char *file, int line, pthread_cond_t *restrict cond, pthread_mutex_t *restrict lock); int debug_mutex_destroy(const char *name, const char *file, int line, pthread_mutex_t *lock); @@ -27,7 +46,7 @@ void debug_dump_lock_stats(); #else -#define mutex_init( lock ) pthread_mutex_init(lock, NULL) +#define mutex_init( lock, prio ) pthread_mutex_init(lock, NULL) #define mutex_lock( lock ) pthread_mutex_lock(lock) #define mutex_trylock( lock ) pthread_mutex_trylock(lock) #define mutex_unlock( lock ) pthread_mutex_unlock(lock) @@ -82,7 +101,4 @@ static inline int debug_thread_join(pthread_t thread, void **value_ptr) #endif -void debug_locks_start_watchdog(); -void debug_locks_stop_watchdog(); - #endif /* LOCKS_H_ */ diff --git a/src/server/net.c b/src/server/net.c index 92728c0..8f97a12 100644 --- a/src/server/net.c +++ b/src/server/net.c @@ -145,7 +145,7 @@ static inline bool sendPadding( const int fd, uint32_t bytes ) void net_init() { - mutex_init( &_clients_lock ); + mutex_init( &_clients_lock, LOCK_CLIENT_LIST ); } void* net_handleNewConnection(void *clientPtr) @@ -186,8 +186,8 @@ void* net_handleNewConnection(void *clientPtr) } } while (0); // Fully init client struct - mutex_init( &client->lock ); - mutex_init( &client->sendMutex ); + mutex_init( &client->lock, LOCK_CLIENT ); + mutex_init( &client->sendMutex, LOCK_CLIENT_SEND ); mutex_lock( &client->lock ); host_to_string( &client->host, client->hostName, HOSTNAMELEN ); diff --git a/src/server/rpc.c b/src/server/rpc.c index 5dbcafe..261c6c0 100644 --- a/src/server/rpc.c +++ b/src/server/rpc.c @@ -75,10 +75,9 @@ static json_int_t randomRunId; static pthread_mutex_t aclLock; #define MAX_CLIENTS 50 #define CUTOFF_START 40 -static pthread_mutex_t statusLock; static struct { - int count; - bool overloaded; + atomic_int count; + atomic_bool overloaded; } status; static bool handleStatus(int sock, int permissions, struct field *fields, size_t fields_num, int keepAlive); @@ -91,8 +90,7 @@ static void loadAcl(); void rpc_init() { - mutex_init( &aclLock ); - mutex_init( &statusLock ); + mutex_init( &aclLock, LOCK_RPC_ACL ); randomRunId = (((json_int_t)getpid()) << 16) | (json_int_t)time(NULL); // if ( sizeof(randomRunId) > 4 ) { @@ -123,10 +121,8 @@ void rpc_sendStatsJson(int sock, dnbd3_host_t* host, const void* data, const int return; } do { - mutex_lock( &statusLock ); const int curCount = ++status.count; UPDATE_LOADSTATE( curCount ); - mutex_unlock( &statusLock ); if ( curCount > MAX_CLIENTS ) { sendReply( sock, "503 Service Temporarily Unavailable", "text/plain", "Too many HTTP clients", -1, HTTP_CLOSE ); goto func_return; @@ -198,9 +194,7 @@ void rpc_sendStatsJson(int sock, dnbd3_host_t* host, const void* data, const int if ( minorVersion == 0 || hasHeaderValue( headers, numHeaders, &STR_CONNECTION, &STR_CLOSE ) ) { keepAlive = HTTP_CLOSE; } else { // And if there aren't too many active HTTP sessions - mutex_lock( &statusLock ); if ( status.overloaded ) keepAlive = HTTP_CLOSE; - mutex_unlock( &statusLock ); } } if ( method.s != NULL && path.s != NULL ) { @@ -234,10 +228,8 @@ void rpc_sendStatsJson(int sock, dnbd3_host_t* host, const void* data, const int } while (true); func_return:; do { - mutex_lock( &statusLock ); const int curCount = --status.count; UPDATE_LOADSTATE( curCount ); - mutex_unlock( &statusLock ); } while (0); } diff --git a/src/server/server.c b/src/server/server.c index 10ab208..838aec2 100644 --- a/src/server/server.c +++ b/src/server/server.c @@ -133,9 +133,6 @@ void dnbd3_cleanup() // Wait for clients to disconnect net_waitForAllDisconnected(); - // Watchdog not needed anymore - debug_locks_stop_watchdog(); - // Clean up images retries = 5; while ( !image_tryFreeAll() && --retries > 0 ) { @@ -303,10 +300,6 @@ int main(int argc, char *argv[]) logadd( LOG_WARNING, "Could not load alt-servers. Does the file exist in %s?", _configDir ); } -#ifdef _DEBUG - debug_locks_start_watchdog(); -#endif - // setup signal handler struct sigaction sa; memset( &sa, 0, sizeof(sa) ); diff --git a/src/server/uplink.c b/src/server/uplink.c index bb1ffdc..9570273 100644 --- a/src/server/uplink.c +++ b/src/server/uplink.c @@ -89,9 +89,9 @@ bool uplink_init(dnbd3_image_t *image, int sock, dnbd3_host_t *host, int version goto failure; } link = image->uplink = calloc( 1, sizeof(dnbd3_connection_t) ); - mutex_init( &link->queueLock ); - mutex_init( &link->rttLock ); - mutex_init( &link->sendMutex ); + mutex_init( &link->queueLock, LOCK_UPLINK_QUEUE ); + mutex_init( &link->rttLock, LOCK_UPLINK_RTT ); + mutex_init( &link->sendMutex, LOCK_UPLINK_SEND ); link->image = image; link->bytesReceived = 0; link->idleTime = 0; -- cgit v1.2.3-55-g7522