Finish modularizing cron; Add error detection for crashed cron jobs

1 files changed, 38 insertions, 16 deletions

diff --git a/apis/cron.inc.php b/apis/cron.inc.php
index 470c70e6..30d6d452 100644
--- a/apis/cron.inc.php
+++ b/apis/cron.inc.php
@@ -10,32 +10,54 @@
 if (!isLocalExecution())
 	exit(0);
 
+define('CRON_KEY_STATUS', 'cron.key.status');
+
+function getJobStatus($id)
+{
+	// Re fetch from D on every call as some jobs could take longer
+	// and we don't want to work with stale data
+	$activeList = Property::getList(CRON_KEY_STATUS);
+	foreach ($activeList as $item) {
+		$entry = explode('|', $item, 2);
+		if (count($entry) !== 2 || $id !== $entry[0])
+			continue;
+		return array('start' => $entry[1], 'string' => $item);
+	}
+	return false;
+}
+
 // Hooks by other modules
 function handleModule($file)
 {
 	include_once $file;
 }
 
-foreach (glob('modules/*/hooks/cron.inc.php') as $file) {
+foreach (glob('modules/*/hooks/cron.inc.php', GLOB_NOSORT) as $file) {
 	preg_match('#^modules/([^/]+)/#', $file, $out);
 	$mod = Module::get($out[1]);
 	if ($mod === false)
 		continue;
+	$id = $mod->getIdentifier();
+	// Check if job is still running, or should be considered crashed
+	$status = getJobStatus($id);
+	if ($status !== false) {
+		$runtime = (time() - $status['start']);
+		if ($runtime < 0) {
+			// Clock skew
+			Property::removeFromList(CRON_KEY_STATUS, $status['string']);
+		} elseif ($runtime < 900) {
+			// Allow up to 15 minutes for a job to complete before we complain...
+			continue;
+		} else {
+			// Consider job crashed
+			Property::removeFromList(CRON_KEY_STATUS, $status['string']);
+			EventLog::failure('Cronjob for module ' . $id . ' seems to be stuck or has crashed. Check the php or web server error log.');
+			continue;
+		}
+	}
+	$now = time();
+	Property::addToList(CRON_KEY_STATUS, "$id|$now", 1800);
 	$mod->activate();
 	handleModule($file);
+	Property::removeFromList(CRON_KEY_STATUS, "$id|$now");
 }
-
-switch (mt_rand(1, 10)) {
-case 2:
-	Database::exec("DELETE FROM property_list WHERE dateline <> 0 AND dateline < UNIX_TIMESTAMP()");
-	break;
-case 3:
-	Database::exec("DELETE FROM property WHERE dateline <> 0 AND dateline < UNIX_TIMESTAMP()");
-	break;
-case 4:
-	Database::exec("DELETE FROM callback WHERE (UNIX_TIMESTAMP() - dateline) > 86400");
-	break;
-}
-
-Trigger::checkCallbacks();
-Trigger::ldadp();