[SERVER] Better lock debugging: Always check lock order

Lock order is predefined in locks.h. Immediately bail out if a lock with
lower priority is obtained while the same thread already holds one with
higher priority.

10 files changed, 192 insertions, 219 deletions

diff --git a/src/server/altservers.c b/src/server/altservers.c
index a270bf3..3d5e71e 100644
--- a/src/server/altservers.c
+++ b/src/server/altservers.c
@@ -30,7 +30,7 @@ void altservers_init()
 {
 	srand( (unsigned int)time( NULL ) );
 	// Init spinlock
-	mutex_init( &altServersLock );
+	mutex_init( &altServersLock, LOCK_ALT_SERVER_LIST );
 	// Init signal
 	runSignal = signal_new();
 	if ( runSignal == NULL ) {
@@ -326,13 +326,13 @@ json_t* altservers_toJson()
 }
 
 /**
- * Update rtt history of given server - returns the new average for that server
+ * Update rtt history of given server - returns the new average for that server.
+ * XXX HOLD altServersLock WHEN CALLING THIS!
  */
 static unsigned int altservers_updateRtt(const dnbd3_host_t * const host, const unsigned int rtt)
 {
 	unsigned int avg = rtt;
 	int i;
-	mutex_lock( &altServersLock );
 	for (i = 0; i < numAltServers; ++i) {
 		if ( !isSameAddressPort( host, &altServers[i].host ) ) continue;
 		altServers[i].rtt[++altServers[i].rttIndex % SERVER_RTT_PROBES] = rtt;
@@ -353,7 +353,6 @@ static unsigned int altservers_updateRtt(const dnbd3_host_t * const host, const
 		}
 		break;
 	}
-	mutex_unlock( &altServersLock );
 	return avg;
 }
 
@@ -529,6 +528,7 @@ static void *altservers_main(void *data UNUSED)
 				}
 				clock_gettime( BEST_CLOCK_SOURCE, &end );
 				// Measurement done - everything fine so far
+				mutex_lock( &altServersLock );
 				mutex_lock( &uplink->rttLock );
 				const bool isCurrent = isSameAddressPort( &servers[itAlt], &uplink->currentServer );
 				// Penaltize rtt if this was a cycle; this will treat this server with lower priority
@@ -538,6 +538,7 @@ static void *altservers_main(void *data UNUSED)
 						+ (end.tv_nsec - start.tv_nsec) / 1000
 						+ ( (isCurrent && uplink->cycleDetected) ? 1000000 : 0 )); // µs
 				unsigned int avg = altservers_updateRtt( &servers[itAlt], rtt );
+				mutex_unlock( &altServersLock );
 				// If a cycle was detected, or we lost connection to the current (last) server, penaltize it one time
 				if ( ( uplink->cycleDetected || uplink->fd == -1 ) && isCurrent ) avg = (avg * 2) + 50000;
 				mutex_unlock( &uplink->rttLock );
diff --git a/src/server/globals.c b/src/server/globals.c
index 69e8a6e..46c1030 100644
--- a/src/server/globals.c
+++ b/src/server/globals.c
@@ -112,7 +112,7 @@ void globals_loadConfig()
 	asprintf( &name, "%s/%s", _configDir, CONFIG_FILENAME );
 	if ( name == NULL ) return;
 	if ( initialLoad ) {
-		mutex_init( &loadLock );
+		mutex_init( &loadLock, LOCK_LOAD_CONFIG );
 	}
 	if ( mutex_trylock( &loadLock ) != 0 ) {
 		logadd( LOG_INFO, "Ignoring config reload request due to already running reload" );
diff --git a/src/server/image.c b/src/server/image.c
index 1f12eda..4a65ed3 100644
--- a/src/server/image.c
+++ b/src/server/image.c
@@ -59,9 +59,9 @@ static bool image_checkRandomBlocks(const int count, int fdImage, const int64_t
 void image_serverStartup()
 {
 	srand( (unsigned int)time( NULL ) );
-	mutex_init( &imageListLock );
-	mutex_init( &remoteCloneLock );
-	mutex_init( &reloadLock );
+	mutex_init( &imageListLock, LOCK_IMAGE_LIST );
+	mutex_init( &remoteCloneLock, LOCK_REMOTE_CLONE );
+	mutex_init( &reloadLock, LOCK_RELOAD );
 }
 
 /**
@@ -347,7 +347,7 @@ dnbd3_image_t* image_get(char *name, uint16_t revision, bool checkIfWorking)
 		img->rid = candidate->rid;
 		img->users = 1;
 		img->working = false;
-		mutex_init( &img->lock );
+		mutex_init( &img->lock, LOCK_IMAGE );
 		if ( candidate->crc32 != NULL ) {
 			const size_t mb = IMGSIZE_TO_HASHBLOCKS( candidate->virtualFilesize ) * sizeof(uint32_t);
 			img->crc32 = malloc( mb );
@@ -869,7 +869,7 @@ static bool image_load(char *base, char *path, int withUplink)
 	image->working = (image->cache_map == NULL );
 	timing_get( &image->nextCompletenessEstimate );
 	image->completenessEstimate = -1;
-	mutex_init( &image->lock );
+	mutex_init( &image->lock, LOCK_IMAGE );
 	int32_t offset;
 	if ( stat( path, &st ) == 0 ) {
 		// Negatively offset atime by file modification time
diff --git a/src/server/integrity.c b/src/server/integrity.c
index a66a364..c52d17b 100644
--- a/src/server/integrity.c
+++ b/src/server/integrity.c
@@ -39,7 +39,7 @@ static void* integrity_main(void *data);
 void integrity_init()
 {
 	assert( queueLen == -1 );
-	mutex_init( &integrityQueueLock );
+	mutex_init( &integrityQueueLock, LOCK_INTEGRITY_QUEUE );
 	pthread_cond_init( &queueSignal, NULL );
 	mutex_lock( &integrityQueueLock );
 	queueLen = 0;
diff --git a/src/server/locks.c b/src/server/locks.c
index 2c0cb27..b39576b 100644
--- a/src/server/locks.c
+++ b/src/server/locks.c
@@ -12,47 +12,45 @@
 #ifdef _DEBUG
 #define MAXLOCKS (SERVER_MAX_CLIENTS * 2 + SERVER_MAX_ALTS + 200 + SERVER_MAX_IMAGES)
 #define MAXTHREADS (SERVER_MAX_CLIENTS + 100)
+#define MAXLPT 20
 #define LOCKLEN 60
 typedef struct
 {
-	void *lock;
+	void * _Atomic lock;
 	ticks locktime;
-	char locked;
-	pthread_t thread;
+	bool _Atomic locked;
+	pthread_t _Atomic thread;
 	int lockId;
+	int prio;
 	char name[LOCKLEN];
 	char where[LOCKLEN];
 } debug_lock_t;
 
 typedef struct
 {
-	pthread_t tid;
+	pthread_t _Atomic tid;
 	ticks time;
 	char name[LOCKLEN];
 	char where[LOCKLEN];
-
+	debug_lock_t *locks[MAXLPT];
 } debug_thread_t;
 
 int debugThreadCount = 0;
 
 static debug_lock_t locks[MAXLOCKS];
 static debug_thread_t threads[MAXTHREADS];
-static int init_done = 0;
-static pthread_mutex_t initdestory;
+static pthread_mutex_t initdestory = PTHREAD_MUTEX_INITIALIZER;
 static int lockId = 0;
-static pthread_t watchdog = 0;
-static dnbd3_signal_t* watchdogSignal = NULL;
 
-static void *debug_thread_watchdog(void *something);
+#define ULDE(...) do { \
+			pthread_mutex_unlock( &initdestory ); \
+			logadd( LOG_ERROR, __VA_ARGS__ ); \
+			debug_dump_lock_stats(); \
+			exit( 4 ); \
+} while(0)
 
-int debug_mutex_init(const char *name, const char *file, int line, pthread_mutex_t *lock)
+int debug_mutex_init(const char *name, const char *file, int line, pthread_mutex_t *lock, int priority)
 {
-	if ( !init_done ) {
-		memset( locks, 0, MAXLOCKS * sizeof(debug_lock_t) );
-		memset( threads, 0, MAXTHREADS * sizeof(debug_thread_t) );
-		pthread_mutex_init( &initdestory, NULL );
-		init_done = 1;
-	}
 	int first = -1;
 	pthread_mutex_lock( &initdestory );
 	for (int i = 0; i < MAXLOCKS; ++i) {
@@ -63,20 +61,18 @@ int debug_mutex_init(const char *name, const char *file, int line, pthread_mutex
 		if ( first == -1 && locks[i].lock == NULL ) first = i;
 	}
 	if ( first == -1 ) {
-		logadd( LOG_ERROR, "No more free debug locks (%s:%d)\n", file, line );
-		pthread_mutex_unlock( &initdestory );
-		debug_dump_lock_stats();
-		exit( 4 );
+		ULDE( "No more free debug locks (%s:%d)\n", file, line );
 	}
 	locks[first].lock = (void*)lock;
-	locks[first].locked = 0;
+	locks[first].locked = false;
+	locks[first].prio = priority;
 	snprintf( locks[first].name, LOCKLEN, "%s", name );
 	snprintf( locks[first].where, LOCKLEN, "I %s:%d", file, line );
 	pthread_mutex_unlock( &initdestory );
 	return pthread_mutex_init( lock, NULL );
 }
 
-int debug_mutex_lock(const char *name, const char *file, int line, pthread_mutex_t *lock)
+int debug_mutex_lock(const char *name, const char *file, int line, pthread_mutex_t *lock, bool try)
 {
 	debug_lock_t *l = NULL;
 	pthread_mutex_lock( &initdestory );
@@ -86,163 +82,180 @@ int debug_mutex_lock(const char *name, const char *file, int line, pthread_mutex
 			break;
 		}
 	}
-	pthread_mutex_unlock( &initdestory );
 	if ( l == NULL ) {
-		logadd( LOG_ERROR, "Tried to lock uninitialized lock %p (%s) at %s:%d\n", (void*)lock, name, file, line );
-		debug_dump_lock_stats();
-		exit( 4 );
+		ULDE( "Tried to lock uninitialized lock %p (%s) at %s:%d\n", (void*)lock, name, file, line );
 	}
 	debug_thread_t *t = NULL;
-	pthread_mutex_lock( &initdestory );
+	int first = -1;
+	const pthread_t self = pthread_self();
 	for (int i = 0; i < MAXTHREADS; ++i) {
-		if ( threads[i].tid != 0 ) continue;
-		threads[i].tid = pthread_self();
-		timing_get( &threads[i].time );
-		snprintf( threads[i].name, LOCKLEN, "%s", name );
-		snprintf( threads[i].where, LOCKLEN, "%s:%d", file, line );
-		t = &threads[i];
-		break;
-	}
-	pthread_mutex_unlock( &initdestory );
-	if ( t == NULL ) {
-		logadd( LOG_ERROR, "Lock sanity check: Too many waiting threads for lock %p (%s) at %s:%d\n", (void*)lock, name, file, line );
-		exit( 4 );
-	}
-	const int retval = pthread_mutex_lock( lock );
-	pthread_mutex_lock( &initdestory );
-	t->tid = 0;
-	pthread_mutex_unlock( &initdestory );
-	if ( l->locked ) {
-		logadd( LOG_ERROR, "Lock sanity check: lock %p (%s) already locked at %s:%d\n", (void*)lock, name, file, line );
-		exit( 4 );
-	}
-	l->locked = 1;
-	timing_get( &l->locktime );
-	l->thread = pthread_self();
-	snprintf( l->where, LOCKLEN, "L %s:%d", file, line );
-	pthread_mutex_lock( &initdestory );
-	l->lockId = ++lockId;
-	pthread_mutex_unlock( &initdestory );
-	return retval;
-}
-
-int debug_mutex_trylock(const char *name, const char *file, int line, pthread_mutex_t *lock)
-{
-	debug_lock_t *l = NULL;
-	pthread_mutex_lock( &initdestory );
-	for (int i = 0; i < MAXLOCKS; ++i) {
-		if ( locks[i].lock == lock ) {
-			l = &locks[i];
+		if ( threads[i].tid == self ) {
+			t = &threads[i];
 			break;
 		}
+		if ( first == -1 && threads[i].tid == 0 ) {
+			first = i;
+		}
 	}
-	pthread_mutex_unlock( &initdestory );
-	if ( l == NULL ) {
-		logadd( LOG_ERROR, "Tried to lock uninitialized lock %p (%s) at %s:%d\n", (void*)lock, name, file, line );
-		debug_dump_lock_stats();
-		exit( 4 );
-	}
-	debug_thread_t *t = NULL;
-	pthread_mutex_lock( &initdestory );
-	for (int i = 0; i < MAXTHREADS; ++i) {
-		if ( threads[i].tid != 0 ) continue;
-		threads[i].tid = pthread_self();
-		timing_get( &threads[i].time );
-		snprintf( threads[i].name, LOCKLEN, "%s", name );
-		snprintf( threads[i].where, LOCKLEN, "%s:%d", file, line );
-		t = &threads[i];
-		break;
-	}
-	pthread_mutex_unlock( &initdestory );
+	int idx;
 	if ( t == NULL ) {
-		logadd( LOG_ERROR, "Lock sanity check: Too many waiting threads for %p (%s) at %s:%d\n", (void*)lock, name, file, line );
-		exit( 4 );
+		if ( first == -1 ) {
+			ULDE( "Lock sanity check: Too many waiting threads for lock %p (%s) at %s:%d\n", (void*)lock, name, file, line );
+		}
+		t = &threads[first];
+		timing_get( &t->time );
+		t->tid = self;
+		snprintf( t->name, LOCKLEN, "%s", name );
+		snprintf( t->where, LOCKLEN, "%s:%d", file, line );
+		memset( t->locks, 0, sizeof(t->locks) );
+		idx = 0;
+	} else {
+		// Thread already has locks, check for order violation
+		idx = -1;
+		for (int i = 0; i < MAXLPT; ++i) {
+			if ( t->locks[i] == NULL ) {
+				if ( idx == -1 ) {
+					idx = i;
+				}
+				continue;
+			}
+			if ( t->locks[i]->prio >= l->prio ) {
+				ULDE( "Lock priority violation: %s at %s:%d (%d) when already holding %s at %s (%d)",
+						name, file, line, l->prio,
+						t->locks[i]->name, t->locks[i]->where, t->locks[i]->prio );
+			}
+			if ( t->locks[i] == l ) {
+				ULDE( "Tried to recusively lock %s in the same thread. Tried at %s:%d, when already locked at %s",
+						name, file, line, t->locks[i]->name );
+			}
+		}
+		if ( idx == -1 ) {
+			ULDE( "Thread %d tried to lock more than %d locks.", (int)self, (int)MAXLPT );
+		}
 	}
-	const int retval = pthread_mutex_trylock( lock );
-	pthread_mutex_lock( &initdestory );
-	t->tid = 0;
 	pthread_mutex_unlock( &initdestory );
+	const int retval = try ? pthread_mutex_trylock( lock ) : pthread_mutex_lock( lock );
 	if ( retval == 0 ) {
+		timing_get( &l->locktime );
+		l->thread = self;
+		snprintf( l->where, LOCKLEN, "L %s:%d", file, line );
+		pthread_mutex_lock( &initdestory );
 		if ( l->locked ) {
 			logadd( LOG_ERROR, "Lock sanity check: lock %p (%s) already locked at %s:%d\n", (void*)lock, name, file, line );
 			exit( 4 );
 		}
-		l->locked = 1;
-		timing_get( &l->locktime );
-		l->thread = pthread_self();
-		snprintf( l->where, LOCKLEN, "L %s:%d", file, line );
-		pthread_mutex_lock( &initdestory );
+		l->locked = true;
+		t->locks[idx] = l;
 		l->lockId = ++lockId;
 		pthread_mutex_unlock( &initdestory );
+	} else if ( !try || retval != EBUSY ) {
+		logadd( LOG_ERROR, "Acquiring lock %s at %s:%d failed with error code %d", name, file, line, retval );
+		debug_dump_lock_stats();
+		exit( 4 );
 	}
 	return retval;
 }
 
 int debug_mutex_unlock(const char *name, const char *file, int line, pthread_mutex_t *lock)
 {
-	debug_lock_t *l = NULL;
+	debug_thread_t *t = NULL;
+	pthread_t self = pthread_self();
 	pthread_mutex_lock( &initdestory );
-	for (int i = 0; i < MAXLOCKS; ++i) {
-		if ( locks[i].lock == lock ) {
-			l = &locks[i];
+	for (int i = 0; i < MAXTHREADS; ++i) {
+		if ( threads[i].tid == self ) {
+			t = &threads[i];
 			break;
 		}
 	}
-	pthread_mutex_unlock( &initdestory );
-	if ( l == NULL ) {
-		logadd( LOG_ERROR, "Tried to unlock uninitialized lock %p (%s) at %s:%d\n", (void*)lock, name, file, line );
-		exit( 4 );
+	if ( t == NULL ) {
+		ULDE( "Unlock called from unknown thread for %s at %s:%d", name, file, line );
 	}
-	if ( !l->locked ) {
-		logadd( LOG_ERROR, "Unlock sanity check: lock %p (%s) not locked at %s:%d\n", (void*)lock, name, file, line );
-		exit( 4 );
+	int idx = -1;
+	int cnt = 0;
+	for (int i = 0; i < MAXLPT; ++i) {
+		if ( t->locks[i] == NULL )
+			continue;
+		cnt++;
+		if ( t->locks[i]->lock == lock ) {
+			idx = i;
+		}
+	}
+	if ( idx == -1 ) {
+		ULDE( "Unlock: Calling thread doesn't hold lock %s at %s:%d", name, file, line );
 	}
-	l->locked = 0;
+	debug_lock_t *l = t->locks[idx];
+	if ( l->thread != self || !l->locked ) {
+		ULDE( "Unlock sanity check for lock debugger failed! Lock %s is assigned to calling thread, but lock's meta data doesn't match up at %s:%d", name, file, line );
+	}
+	l->locked = false;
 	l->thread = 0;
+	t->locks[idx] = NULL;
+	if ( cnt == 1 ) {
+		t->tid = 0; // No more locks held, free up slot
+	}
 	snprintf( l->where, LOCKLEN, "U %s:%d", file, line );
-	int retval = pthread_mutex_unlock( lock );
+	pthread_mutex_unlock( &initdestory );
+	const int retval = pthread_mutex_unlock( lock );
+	if ( retval != 0 ) {
+		logadd( LOG_ERROR, "pthread_mutex_unlock returned %d for %s at %s:%d", retval, name, file, line );
+		exit( 4 );
+	}
 	return retval;
 }
 
 int debug_mutex_cond_wait(const char *name, const char *file, int line, pthread_cond_t *restrict cond, pthread_mutex_t *restrict lock)
 {
 	debug_lock_t *l = NULL;
+	debug_thread_t *t = NULL;
+	pthread_t self = pthread_self();
 	pthread_mutex_lock( &initdestory );
-	for (int i = 0; i < MAXLOCKS; ++i) {
-		if ( locks[i].lock == lock ) {
-			l = &locks[i];
+	for (int i = 0; i < MAXTHREADS; ++i) {
+		if ( threads[i].tid == self ) {
+			t = &threads[i];
 			break;
 		}
 	}
-	pthread_mutex_unlock( &initdestory );
+	if ( t == NULL ) {
+		ULDE( "Unlock called from unknown thread for %s at %s:%d", name, file, line );
+	}
+	int mp = 0, mpi = -1;
+	for (int i = 0; i < MAXLPT; ++i) {
+		if ( t->locks[i] == NULL )
+			continue;
+		if ( t->locks[i]->lock == lock ) {
+			l = t->locks[i];
+		} else if ( t->locks[i]->prio > mp ) {
+			mp = t->locks[i]->prio;
+			mpi = i;
+		}
+	}
 	if ( l == NULL ) {
-		logadd( LOG_ERROR, "Tried to cond_wait on uninitialized lock %p (%s) at %s:%d\n", (void*)lock, name, file, line );
-		exit( 4 );
+		ULDE( "cond_wait: Calling thread doesn't hold lock %s at %s:%d", name, file, line );
 	}
-	if ( !l->locked ) {
-		logadd( LOG_ERROR, "Cond_wait sanity check: lock %p (%s) not locked at %s:%d\n", (void*)lock, name, file, line );
-		exit( 4 );
+	if ( l->thread != self || !l->locked ) {
+		ULDE( "cond_wait: Sanity check for lock debugger failed! Lock %s is assigned to calling thread, but lock's meta data doesn't match up at %s:%d", name, file, line );
 	}
-	pthread_t self = pthread_self();
-	if ( l->thread != self ) {
-		logadd( LOG_ERROR, "Cond_wait called from non-owning thread for %p (%s) at %s:%d\n", (void*)lock, name, file, line );
-		exit( 4 );
+	if ( mp >= l->prio ) {
+		ULDE( "cond_wait: Yielding a mutex while holding another one with higher prio: %s at %s:%d (%d) while also holding %s at %s (%d)",
+				name, file, line, l->prio,
+				t->locks[mpi]->name, t->locks[mpi]->where, mp );
 	}
-	l->locked = 0;
+	l->locked = false;
 	l->thread = 0;
-	snprintf( l->where, LOCKLEN, "CW %s:%d", file, line );
+	snprintf( l->where, LOCKLEN, "CWU %s:%d", file, line );
+	pthread_mutex_unlock( &initdestory );
 	int retval = pthread_cond_wait( cond, lock );
 	if ( retval != 0 ) {
 		logadd( LOG_ERROR, "pthread_cond_wait returned %d for lock %p (%s) at %s:%d\n", retval, (void*)lock, name, file, line );
 		exit( 4 );
 	}
-	if ( l->locked != 0 || l->thread != 0 ) {
+	if ( l->locked || l->thread != 0 ) {
 		logadd( LOG_ERROR, "Lock is not free after returning from pthread_cond_wait for %p (%s) at %s:%d\n", (void*)lock, name, file, line );
 		exit( 4 );
 	}
-	l->locked = 1;
 	l->thread = self;
 	timing_get( &l->locktime );
+	l->locked = true;
 	pthread_mutex_lock( &initdestory );
 	l->lockId = ++lockId;
 	pthread_mutex_unlock( &initdestory );
@@ -290,63 +303,21 @@ void debug_dump_lock_stats()
 					"* Locked: %d\n", locks[i].name, locks[i].where, (int)locks[i].locked );
 		}
 	}
-	printf( "\n **** WAITING THREADS ****\n\n" );
+	printf( "\n **** ACTIVE THREADS ****\n\n" );
 	for (int i = 0; i < MAXTHREADS; ++i) {
-		if ( threads[i].tid == 0 ) continue;
+		if ( threads[i].tid == 0 )
+			continue;
 		printf( "* *** Thread %d ***\n"
 				"* Lock: %s\n"
 				"* Where: %s\n"
 				"* How long: %d secs\n", (int)threads[i].tid, threads[i].name, threads[i].where, (int)timing_diff( &threads[i].time, &now ) );
-	}
-	pthread_mutex_unlock( &initdestory );
-}
-
-static void *debug_thread_watchdog(void *something UNUSED)
-{
-	setThreadName( "debug-watchdog" );
-	while ( !_shutdown ) {
-		if ( init_done ) {
-			declare_now;
-			pthread_mutex_lock( &initdestory );
-			for (int i = 0; i < MAXTHREADS; ++i) {
-				if ( threads[i].tid == 0 ) continue;
-				const uint32_t diff = timing_diff( &threads[i].time, &now );
-				if ( diff > 6 && diff < 100000 ) {
-					printf( "\n\n +++++++++ DEADLOCK ++++++++++++\n\n" );
-					pthread_mutex_unlock( &initdestory );
-					debug_dump_lock_stats();
-					exit( 99 );
-				}
-			}
-			pthread_mutex_unlock( &initdestory );
+		for (int j = 0; j < MAXLPT; ++j) {
+			if ( threads[i].locks[j] == NULL )
+				continue;
+			printf( "  * Lock %s @ %s\n", threads[i].locks[j]->name, threads[i].locks[j]->where );
 		}
-		if ( watchdogSignal == NULL || signal_wait( watchdogSignal, 5000 ) == SIGNAL_ERROR ) sleep( 5 );
 	}
-	return NULL ;
-}
-
-#endif
-
-void debug_locks_start_watchdog()
-{
-#ifdef _DEBUG
-	watchdogSignal = signal_new();
-	if ( 0 != thread_create( &watchdog, NULL, &debug_thread_watchdog, (void *)NULL ) ) {
-		logadd( LOG_ERROR, "Could not start debug-lock watchdog." );
-		return;
-	}
-#endif
+	pthread_mutex_unlock( &initdestory );
 }
 
-void debug_locks_stop_watchdog()
-{
-#ifdef _DEBUG
-	_shutdown = true;
-	printf( "Killing debug watchdog...\n" );
-	pthread_mutex_lock( &initdestory );
-	signal_call( watchdogSignal );
-	pthread_mutex_unlock( &initdestory );
-	thread_join( watchdog, NULL );
-	signal_close( watchdogSignal );
 #endif
-}
diff --git a/src/server/locks.h b/src/server/locks.h
index 7f72722..e5c9801 100644
--- a/src/server/locks.h
+++ b/src/server/locks.h
@@ -5,19 +5,38 @@
 #include <errno.h>
 #include <stdio.h>
 #include <stdlib.h>
+#include <stdbool.h>
+
+// Lock priority
+
+#define LOCK_RELOAD 90
+#define LOCK_LOAD_CONFIG 100
+#define LOCK_REMOTE_CLONE 110
+#define LOCK_CLIENT_LIST 120
+#define LOCK_CLIENT 130
+#define LOCK_INTEGRITY_QUEUE 140
+#define LOCK_IMAGE_LIST 150
+#define LOCK_IMAGE 160
+#define LOCK_UPLINK_QUEUE 170
+#define LOCK_ALT_SERVER_LIST 180
+#define LOCK_CLIENT_SEND 190
+#define LOCK_UPLINK_RTT 200
+#define LOCK_UPLINK_SEND 210
+#define LOCK_RPC_ACL 220
+
+//
 
 #ifdef _DEBUG
 
-#define mutex_init( lock ) debug_mutex_init( #lock, __FILE__, __LINE__, lock)
-#define mutex_lock( lock ) debug_mutex_lock( #lock, __FILE__, __LINE__, lock)
-#define mutex_trylock( lock ) debug_mutex_trylock( #lock, __FILE__, __LINE__, lock)
+#define mutex_init( lock, prio ) debug_mutex_init( #lock, __FILE__, __LINE__, lock, prio)
+#define mutex_lock( lock ) debug_mutex_lock( #lock, __FILE__, __LINE__, lock, false)
+#define mutex_trylock( lock ) debug_mutex_lock( #lock, __FILE__, __LINE__, lock, true)
 #define mutex_unlock( lock ) debug_mutex_unlock( #lock, __FILE__, __LINE__, lock)
 #define mutex_cond_wait( cond, lock ) debug_mutex_cond_wait( #lock, __FILE__, __LINE__, cond, lock)
 #define mutex_destroy( lock ) debug_mutex_destroy( #lock, __FILE__, __LINE__, lock)
 
-int debug_mutex_init(const char *name, const char *file, int line, pthread_mutex_t *lock);
-int debug_mutex_lock(const char *name, const char *file, int line, pthread_mutex_t *lock);
-int debug_mutex_trylock(const char *name, const char *file, int line, pthread_mutex_t *lock);
+int debug_mutex_init(const char *name, const char *file, int line, pthread_mutex_t *lock, int priority);
+int debug_mutex_lock(const char *name, const char *file, int line, pthread_mutex_t *lock, bool try);
 int debug_mutex_unlock(const char *name, const char *file, int line, pthread_mutex_t *lock);
 int debug_mutex_cond_wait(const char *name, const char *file, int line, pthread_cond_t *restrict cond, pthread_mutex_t *restrict lock);
 int debug_mutex_destroy(const char *name, const char *file, int line, pthread_mutex_t *lock);
@@ -27,7 +46,7 @@ void debug_dump_lock_stats();
 
 #else
 
-#define mutex_init( lock ) pthread_mutex_init(lock, NULL)
+#define mutex_init( lock, prio ) pthread_mutex_init(lock, NULL)
 #define mutex_lock( lock ) pthread_mutex_lock(lock)
 #define mutex_trylock( lock ) pthread_mutex_trylock(lock)
 #define mutex_unlock( lock ) pthread_mutex_unlock(lock)
@@ -82,7 +101,4 @@ static inline int debug_thread_join(pthread_t thread, void **value_ptr)
 
 #endif
 
-void debug_locks_start_watchdog();
-void debug_locks_stop_watchdog();
-
 #endif /* LOCKS_H_ */
diff --git a/src/server/net.c b/src/server/net.c
index 92728c0..8f97a12 100644
--- a/src/server/net.c
+++ b/src/server/net.c
@@ -145,7 +145,7 @@ static inline bool sendPadding( const int fd, uint32_t bytes )
 
 void net_init()
 {
-	mutex_init( &_clients_lock );
+	mutex_init( &_clients_lock, LOCK_CLIENT_LIST );
 }
 
 void* net_handleNewConnection(void *clientPtr)
@@ -186,8 +186,8 @@ void* net_handleNewConnection(void *clientPtr)
 		}
 	} while (0);
 	// Fully init client struct
-	mutex_init( &client->lock );
-	mutex_init( &client->sendMutex );
+	mutex_init( &client->lock, LOCK_CLIENT );
+	mutex_init( &client->sendMutex, LOCK_CLIENT_SEND );
 
 	mutex_lock( &client->lock );
 	host_to_string( &client->host, client->hostName, HOSTNAMELEN );
diff --git a/src/server/rpc.c b/src/server/rpc.c
index 5dbcafe..261c6c0 100644
--- a/src/server/rpc.c
+++ b/src/server/rpc.c
@@ -75,10 +75,9 @@ static json_int_t randomRunId;
 static pthread_mutex_t aclLock;
 #define MAX_CLIENTS 50
 #define CUTOFF_START 40
-static pthread_mutex_t statusLock;
 static struct {
-	int count;
-	bool overloaded;
+	atomic_int count;
+	atomic_bool overloaded;
 } status;
 
 static bool handleStatus(int sock, int permissions, struct field *fields, size_t fields_num, int keepAlive);
@@ -91,8 +90,7 @@ static void loadAcl();
 
 void rpc_init()
 {
-	mutex_init( &aclLock );
-	mutex_init( &statusLock );
+	mutex_init( &aclLock, LOCK_RPC_ACL );
 	randomRunId = (((json_int_t)getpid()) << 16) | (json_int_t)time(NULL);
 	// </guard>
 	if ( sizeof(randomRunId) > 4 ) {
@@ -123,10 +121,8 @@ void rpc_sendStatsJson(int sock, dnbd3_host_t* host, const void* data, const int
 		return;
 	}
 	do {
-		mutex_lock( &statusLock );
 		const int curCount = ++status.count;
 		UPDATE_LOADSTATE( curCount );
-		mutex_unlock( &statusLock );
 		if ( curCount > MAX_CLIENTS ) {
 			sendReply( sock, "503 Service Temporarily Unavailable", "text/plain", "Too many HTTP clients", -1, HTTP_CLOSE );
 			goto func_return;
@@ -198,9 +194,7 @@ void rpc_sendStatsJson(int sock, dnbd3_host_t* host, const void* data, const int
 			if ( minorVersion == 0 || hasHeaderValue( headers, numHeaders, &STR_CONNECTION, &STR_CLOSE ) ) {
 				keepAlive = HTTP_CLOSE;
 			} else { // And if there aren't too many active HTTP sessions
-				mutex_lock( &statusLock );
 				if ( status.overloaded ) keepAlive = HTTP_CLOSE;
-				mutex_unlock( &statusLock );
 			}
 		}
 		if ( method.s != NULL && path.s != NULL ) {
@@ -234,10 +228,8 @@ void rpc_sendStatsJson(int sock, dnbd3_host_t* host, const void* data, const int
 	} while (true);
 func_return:;
 	do {
-		mutex_lock( &statusLock );
 		const int curCount = --status.count;
 		UPDATE_LOADSTATE( curCount );
-		mutex_unlock( &statusLock );
 	} while (0);
 }
 
diff --git a/src/server/server.c b/src/server/server.c
index 10ab208..838aec2 100644
--- a/src/server/server.c
+++ b/src/server/server.c
@@ -133,9 +133,6 @@ void dnbd3_cleanup()
 	// Wait for clients to disconnect
 	net_waitForAllDisconnected();
 
-	// Watchdog not needed anymore
-	debug_locks_stop_watchdog();
-
 	// Clean up images
 	retries = 5;
 	while ( !image_tryFreeAll() && --retries > 0 ) {
@@ -303,10 +300,6 @@ int main(int argc, char *argv[])
 		logadd( LOG_WARNING, "Could not load alt-servers. Does the file exist in %s?", _configDir );
 	}
 
-#ifdef _DEBUG
-	debug_locks_start_watchdog();
-#endif
-
 	// setup signal handler
 	struct sigaction sa;
 	memset( &sa, 0, sizeof(sa) );
diff --git a/src/server/uplink.c b/src/server/uplink.c
index bb1ffdc..9570273 100644
--- a/src/server/uplink.c
+++ b/src/server/uplink.c
@@ -89,9 +89,9 @@ bool uplink_init(dnbd3_image_t *image, int sock, dnbd3_host_t *host, int version
 		goto failure;
 	}
 	link = image->uplink = calloc( 1, sizeof(dnbd3_connection_t) );
-	mutex_init( &link->queueLock );
-	mutex_init( &link->rttLock );
-	mutex_init( &link->sendMutex );
+	mutex_init( &link->queueLock, LOCK_UPLINK_QUEUE );
+	mutex_init( &link->rttLock, LOCK_UPLINK_RTT );
+	mutex_init( &link->sendMutex, LOCK_UPLINK_SEND );
 	link->image = image;
 	link->bytesReceived = 0;
 	link->idleTime = 0;