summaryrefslogtreecommitdiffstats
path: root/modules-available/rebootcontrol/inc/rebootcontrol.inc.php
diff options
context:
space:
mode:
Diffstat (limited to 'modules-available/rebootcontrol/inc/rebootcontrol.inc.php')
-rw-r--r--modules-available/rebootcontrol/inc/rebootcontrol.inc.php411
1 files changed, 381 insertions, 30 deletions
diff --git a/modules-available/rebootcontrol/inc/rebootcontrol.inc.php b/modules-available/rebootcontrol/inc/rebootcontrol.inc.php
index 8a85e3ff..489b0252 100644
--- a/modules-available/rebootcontrol/inc/rebootcontrol.inc.php
+++ b/modules-available/rebootcontrol/inc/rebootcontrol.inc.php
@@ -5,9 +5,14 @@ class RebootControl
const KEY_TASKLIST = 'rebootcontrol.tasklist';
+ const KEY_AUTOSCAN_DISABLED = 'rebootcontrol.disable.scan';
+
const REBOOT = 'REBOOT';
const KEXEC_REBOOT = 'KEXEC_REBOOT';
const SHUTDOWN = 'SHUTDOWN';
+ const TASK_REBOOTCTL = 'TASK_REBOOTCTL';
+ const TASK_WOL = 'WAKE_ON_LAN';
+ const TASK_EXEC = 'REMOTE_EXEC';
/**
* @param string[] $uuids List of machineuuids to reboot
@@ -19,66 +24,136 @@ class RebootControl
$list = RebootQueries::getMachinesByUuid($uuids);
if (empty($list))
return false;
- return self::execute($list, $kexec ? RebootControl::KEXEC_REBOOT : RebootControl::REBOOT, 0, 0);
+ return self::execute($list, $kexec ? RebootControl::KEXEC_REBOOT : RebootControl::REBOOT, 0);
}
/**
- * @param array $list list of clients containing each keys 'machineuuid' and 'clientip'
+ * @param array $list list of clients containing each keys 'machineuuid', 'clientip' and 'locationid'
* @param string $mode reboot mode: RebootControl::REBOOT ::KEXEC_REBOOT or ::SHUTDOWN
* @param int $minutes delay in minutes for action
* @param int $locationId meta data only: locationId of clients
* @return array|false the task, or false if it could not be started
*/
- public static function execute($list, $mode, $minutes, $locationId)
+ public static function execute($list, $mode, $minutes)
{
$task = Taskmanager::submit("RemoteReboot", array(
"clients" => $list,
"mode" => $mode,
"minutes" => $minutes,
- "locationId" => $locationId,
"sshkey" => SSHKey::getPrivateKey(),
"port" => 9922, // Hard-coded, must match mgmt-sshd module
));
if (!Taskmanager::isFailed($task)) {
- Property::addToList(RebootControl::KEY_TASKLIST, $locationId . '/' . $task["id"], 60 * 24);
+ self::addTask($task['id'], self::TASK_REBOOTCTL, $list, $task['id'], ['action' => $mode]);
}
return $task;
}
+ private static function extractLocationIds($from, &$to)
+ {
+ if (is_numeric($from)) {
+ $to[$from] = true;
+ return;
+ }
+ if (!is_array($from))
+ return;
+ $allnum = true;
+ foreach ($from as $k => $v) {
+ if (is_numeric($k) && is_numeric($v))
+ continue;
+ $allnum = false;
+ if (is_numeric($k) && is_array($v)) {
+ self::extractLocationIds($v, $to);
+ } else {
+ $k = strtolower($k);
+ if ($k === 'locationid' || $k === 'locationids' || $k === 'location' || $k === 'locations' || $k === 'lid' || $k === 'lids') {
+ self::extractLocationIds($v, $to);
+ } elseif ($k === 'client' || $k === 'clients' || $k === 'machine' || $k === 'machines') {
+ if (is_array($v)) {
+ self::extractLocationIds($v, $to);
+ }
+ }
+ }
+ }
+ if ($allnum) {
+ foreach ($from as $v) {
+ $to[$v] = true;
+ }
+ }
+ }
+
+ private static function addTask($id, $type, $clients, $taskIds, $other = false)
+ {
+ $lids = ArrayUtil::flattenByKey($clients, 'locationid');
+ $lids = array_unique($lids);
+ $newClients = [];
+ foreach ($clients as $c) {
+ $d = ['clientip' => $c['clientip']];
+ if (isset($c['machineuuid'])) {
+ $d['machineuuid'] = $c['machineuuid'];
+ }
+ $newClients[] = $d;
+ }
+ if (!is_array($taskIds)) {
+ $taskIds = [$taskIds];
+ }
+ $data = [
+ 'id' => $id,
+ 'type' => $type,
+ 'locations' => $lids,
+ 'clients' => $newClients,
+ 'tasks' => $taskIds,
+ ];
+ if (is_array($other)) {
+ $data += $other;
+ }
+ Property::addToList(RebootControl::KEY_TASKLIST, json_encode($data), 20);
+ }
+
/**
* @param int[]|null $locations filter by these locations
- * @return array list of active tasks for reboots/shutdowns.
+ * @return array|false list of active tasks for reboots/shutdowns.
*/
- public static function getActiveTasks($locations = null)
+ public static function getActiveTasks($locations = null, $id = null)
{
- if (is_array($locations) && in_array(0,$locations)) {
+ if (is_array($locations) && in_array(0, $locations)) {
$locations = null;
}
$list = Property::getList(RebootControl::KEY_TASKLIST);
$return = [];
foreach ($list as $entry) {
- $p = explode('/', $entry, 2);
- if (count($p) !== 2) {
+ $p = json_decode($entry, true);
+ if (!is_array($p) || !isset($p['id'])) {
Property::removeFromList(RebootControl::KEY_TASKLIST, $entry);
continue;
}
- if (is_array($locations) && !in_array($p[0], $locations)) // Ignore
+ if (is_array($locations) && is_array($p['locations']) && array_diff($p['locations'], $locations) !== [])
+ continue; // Not allowed
+ if ($id !== null) {
+ if ($p['id'] === $id)
+ return $p;
continue;
- $id = $p[1];
- $task = Taskmanager::status($id);
- if (!Taskmanager::isTask($task)) {
+ }
+ $valid = empty($p['tasks']);
+ if (!$valid) {
+ // Validate at least one task is still valid
+ foreach ($p['tasks'] as $task) {
+ $task = Taskmanager::status($task);
+ if (Taskmanager::isTask($task)) {
+ $p['status'] = $task['statusCode'];
+ $valid = true;
+ break;
+ }
+ }
+ }
+ if (!$valid) {
Property::removeFromList(RebootControl::KEY_TASKLIST, $entry);
continue;
}
- $return[] = [
- 'taskId' => $task['id'],
- 'locationId' => $task['data']['locationId'],
- 'time' => $task['data']['time'],
- 'mode' => $task['data']['mode'],
- 'clientCount' => count($task['data']['clients']),
- 'status' => $task['statusCode'],
- ];
+ $return[] = $p;
}
+ if ($id !== null)
+ return false;
return $return;
}
@@ -96,6 +171,15 @@ class RebootControl
*/
public static function runScript($clients, $command, $timeout = 5, $privkey = false)
{
+ $task = self::runScriptInternal($clients, $command, $timeout, $privkey);
+ if (!Taskmanager::isFailed($task)) {
+ self::addTask($task['id'], self::TASK_EXEC, $clients, $task['id']);
+ }
+ return $task;
+ }
+
+ private static function runScriptInternal(&$clients, $command, $timeout = 5, $privkey = false)
+ {
$valid = [];
$invalid = [];
foreach ($clients as $client) {
@@ -110,7 +194,7 @@ class RebootControl
}
}
if (!empty($invalid)) {
- $res = Database::simpleQuery('SELECT machineuuid, clientip FROM machine WHERE machineuuid IN (:uuids)',
+ $res = Database::simpleQuery('SELECT machineuuid, clientip, locationid FROM machine WHERE machineuuid IN (:uuids)',
['uuids' => array_keys($invalid)]);
while ($row = $res->fetch(PDO::FETCH_ASSOC)) {
if (isset($invalid[$row['machineuuid']])) {
@@ -120,20 +204,21 @@ class RebootControl
}
}
}
+ $clients = $valid;
+ if (empty($clients)) {
+ error_log('RebootControl::runScript called without any clients');
+ return false;
+ }
if ($privkey === false) {
$privkey = SSHKey::getPrivateKey();
}
- $task = Taskmanager::submit('RemoteExec', [
- 'clients' => $valid,
+ return Taskmanager::submit('RemoteExec', [
+ 'clients' => $clients,
'command' => $command,
'timeoutSeconds' => $timeout,
'sshkey' => $privkey,
'port' => 9922, // Fallback if no port given in client struct
]);
- if (!Taskmanager::isFailed($task)) {
- Property::addToList(RebootControl::KEY_TASKLIST, '0/' . $task["id"], 60 * 24);
- }
- return $task;
}
public static function connectionCheckCallback($task, $hostId)
@@ -150,4 +235,270 @@ class RebootControl
['id' => $hostId, 'reachable' => $reachable]);
}
-} \ No newline at end of file
+ /**
+ * @param array $sourceMachines list of source machines. array of [clientip, machineuuid] entries
+ * @param string $bcast directed broadcast address to send to
+ * @param string|string[] $macaddr destination mac address(es)
+ * @param string $passwd optional WOL password, mac address or ipv4 notation
+ * @return array|false task struct, false on error
+ */
+ public static function wakeViaClient($sourceMachines, $macaddr, $bcast = false, $passwd = false)
+ {
+ $command = 'jawol';
+ if (!empty($bcast)) {
+ $command .= " -d '$bcast'";
+ }
+ if (!empty($passwd)) {
+ $command .= " -p '$passwd'";
+ }
+ if (is_array($macaddr)) {
+ $macaddr = implode("' '", $macaddr);
+ }
+ $command .= " '$macaddr'";
+ // Yes there is one zero missing from the usleep -- that's the whole point: we prefer 100ms sleeps
+ return self::runScriptInternal($sourceMachines,
+ "for i in 1 1 0; do $command; usleep \${i}00000 2> /dev/null || sleep \$i; done");
+ }
+
+ /**
+ * @param string|string[] $macaddr destination mac address(es)
+ * @param string $bcast directed broadcast address to send to
+ * @param string $passwd optional WOL password, mac address or ipv4 notation
+ * @return array|false task struct, false on error
+ */
+ public static function wakeDirectly($macaddr, $bcast = false, $passwd = false)
+ {
+ if (!is_array($macaddr)) {
+ $macaddr = [$macaddr];
+ }
+ return Taskmanager::submit('WakeOnLan', [
+ 'ip' => $bcast,
+ 'password' => $passwd,
+ 'macs' => $macaddr,
+ ]);
+ }
+
+ public static function wakeViaJumpHost($jumphost, $bcast, $clients)
+ {
+ $hostid = $jumphost['hostid'];
+ $macs = ArrayUtil::flattenByKey($clients, 'macaddr');
+ if (empty($macs)) {
+ error_log('Called wakeViaJumpHost without clients');
+ return false;
+ }
+ $macs = "'" . implode("' '", $macs) . "'";
+ $macs = str_replace('-', ':', $macs);
+ $script = str_replace(['%IP%', '%MACS%'], [$bcast, $macs], $jumphost['script']);
+ $task = RebootControl::runScriptInternal($_ = [[
+ 'clientip' => $jumphost['host'],
+ 'port' => $jumphost['port'],
+ 'username' => $jumphost['username'],
+ ]], $script, 6, $jumphost['sshkey']);
+ if ($task !== false && isset($task['id'])) {
+ TaskmanagerCallback::addCallback($task, 'rbcConnCheck', $hostid);
+ }
+ return $task;
+ }
+
+ /**
+ * @param array $list list of clients containing each keys 'macaddr' and 'clientip'
+ * @return string id of this job
+ */
+ public static function wakeMachines($list)
+ {
+ /* TODO: Refactor mom's spaghetti
+ * Now that I figured out what I want, do something like this:
+ * 1) Group clients by subnet
+ * 2) Only after step 1, start to collect possible ways to wake up clients for each subnet that's not empty
+ * 3) Habe some priority list for the methods, extend Taskmanager to have "negative dependency"
+ * i.e. submit task B with task A as parent task, but only launch task B if task A failed.
+ * If task A succeeded, mark task B as FINISHED immediately without actually running it.
+ * (or introduce new statusCode for this?)
+ */
+ $errors = '';
+ $tasks = [];
+ $bad = $unknown = [];
+ // Need all subnets...
+ $subnets = [];
+ $res = Database::simpleQuery('SELECT subnetid, start, end, isdirect FROM reboot_subnet');
+ while ($row = $res->fetch(PDO::FETCH_ASSOC)) {
+ $row += [
+ 'jumphosts' => [],
+ 'direct' => [],
+ 'indirect' => [],
+ ];
+ $subnets[$row['subnetid']] = $row;
+ }
+ // Get all jump hosts
+ $jumphosts = [];
+ $res = Database::simpleQuery('SELECT jh.hostid, host, port, username, sshkey, script, jh.reachable,
+ Group_Concat(jxs.subnetid) AS subnets1, Group_Concat(sxs.dstid) AS subnets2
+ FROM reboot_jumphost jh
+ LEFT JOIN reboot_jumphost_x_subnet jxs ON (jh.hostid = jxs.hostid)
+ LEFT JOIN reboot_subnet s ON (INET_ATON(jh.host) BETWEEN s.start AND s.end)
+ LEFT JOIN reboot_subnet_x_subnet sxs ON (sxs.srcid = s.subnetid AND sxs.reachable <> 0)
+ GROUP BY jh.hostid');
+ while ($row = $res->fetch(PDO::FETCH_ASSOC)) {
+ if ($row['subnets1'] === null && $row['subnets2'] === null)
+ continue;
+ $nets = explode(',', $row['subnets1'] . ',' . $row['subnets2']);
+ foreach ($nets as $net) {
+ if (empty($net) || !isset($subnets[$net]))
+ continue;
+ $subnets[$net]['jumphosts'][$row['hostid']] = $row['hostid'];
+ }
+ $row['jobs'] = [];
+ $jumphosts[] = $row;
+ }
+ // Group by subnet
+ foreach ($list as $client) {
+ $ip = sprintf('%u', ip2long($client['clientip']));
+ //$client['numip'] = $ip;
+ unset($subnet);
+ $subnet = false;
+ foreach ($subnets as &$sn) {
+ if ($sn['start'] <= $ip && $sn['end'] >= $ip) {
+ $subnet =& $sn;
+ break;
+ }
+ }
+ $ok = false;
+ if (!$ok && $subnet === false) {
+ $unknown[] = $client;
+ $ok = true;
+ }
+ if (!$ok && $subnet['isdirect']) {
+ // Directly reachable
+ $subnet['direct'][] = $client;
+ $ok = true;
+ }
+ if (!$ok && !empty($subnet['jumphosts'])) {
+ foreach ($subnet['jumphosts'] as $hostid) {
+ if ($jumphosts[$hostid]['reachable'] != 0) {
+ $jumphosts[$hostid]['jobs'][$subnet['end']][] = $client;
+ $ok = true;
+ break;
+ }
+ }
+ }
+ if (!$ok) {
+ // find clients in same subnet, or reachable ones
+ self::findMachinesForSubnet($subnet);
+ if (empty($subnet['dclients']) && empty($subnet['iclients'])) {
+ // Nothing found -- cannot wake this host
+ $bad[] = $client;
+ } else {
+ // Found suitable indirect host
+ $subnet['indirect'][] = $client;
+ }
+ }
+ }
+ unset($subnet);
+ // Batch process
+ // First, via jump host
+ foreach ($jumphosts as $jh) {
+ foreach ($jh['jobs'] as $bcast => $clients) {
+ $errors .= 'Via jumphost ' . $jh['host'] . ': ' . implode(', ', ArrayUtil::flattenByKey($clients, 'clientip')) . "\n";
+ $task = self::wakeViaJumpHost($jh, $bcast, $clients);
+ if (Taskmanager::isFailed($task)) {
+ // TODO: Figure out $subnet from $bcast and queue as indirect
+ // (rather, overhaul this whole spaghetti code)
+ $errors .= ".... FAILED TO LAUNCH TASK ON JUMPHOST!\n";
+ }
+ }
+ }
+ // Server or client
+ foreach ($subnets as $subnet) {
+ if (!empty($subnet['direct'])) {
+ // Can wake directly
+ if (!self::wakeGroup('From server', $tasks, $errors, null, $subnet['direct'], $subnet['end'])) {
+ if (!empty($subnet['dclients']) || !empty($subnet['iclients'])) {
+ $errors .= "Re-queueing clients for indirect wakeup\n";
+ $subnet['indirect'] = array_merge($subnet['indirect'], $subnet['direct']);
+ }
+ }
+ }
+ if (!empty($subnet['indirect'])) {
+ // Can wake indirectly
+ $ok = false;
+ if (!empty($subnet['dclients'])) {
+ $ok = true;
+ if (!self::wakeGroup('in same subnet', $tasks, $errors, $subnet['dclients'], $subnet['indirect'])) {
+ if (!empty($subnet['iclients'])) {
+ $errors .= "Re-re-queueing clients for indirect wakeup\n";
+ $ok = false;
+ }
+ }
+ }
+ if (!$ok && !empty($subnet['iclients'])) {
+ $ok = self::wakeGroup('across subnets', $tasks, $errors, $subnet['dclients'], $subnet['indirect'], $subnet['end']);
+ }
+ if (!$ok) {
+ $errors .= "I'm all out of ideas.\n";
+ }
+ }
+ }
+ if (!empty($bad)) {
+ $ips = ArrayUtil::flattenByKey($bad, 'clientip');
+ $errors .= "**** WARNING ****\nNo way to send WOL packets to the following machines:\n" . implode("\n", $ips) . "\n";
+ }
+ if (!empty($unknown)) {
+ $ips = ArrayUtil::flattenByKey($unknown, 'clientip');
+ $errors .= "**** WARNING ****\nThe following clients do not belong to a known subnet (bug?)\n" . implode("\n", $ips) . "\n";
+ }
+ $id = Util::randomUuid();
+ self::addTask($id, self::TASK_WOL, $list, $tasks, ['log' => $errors]);
+ return $id;
+ }
+
+ private static function wakeGroup($type, &$tasks, &$errors, $via, $clients, $bcast = false)
+ {
+ $macs = ArrayUtil::flattenByKey($clients, 'macaddr');
+ $ips = ArrayUtil::flattenByKey($clients, 'clientip');
+ if ($via !== null) {
+ $srcips = ArrayUtil::flattenByKey($via, 'clientip');
+ $errors .= 'Via ' . implode(', ', $srcips) . ' ';
+ }
+ $errors .= $type . ': ' . implode(', ', $ips);
+ if ($bcast !== false) {
+ $errors .= ' (UDP to ' . long2ip($bcast) . ')';
+ }
+ $errors .= "\n";
+ if ($via === null) {
+ $task = self::wakeDirectly($macs, $bcast);
+ } else {
+ $task = self::wakeViaClient($via, $macs, $bcast);
+ }
+ if ($task !== false && isset($task['id'])) {
+ $tasks[] = $task['id'];
+ }
+ if (Taskmanager::isFailed($task)) {
+ $errors .= ".... FAILED TO START ACCORDING TASK!\n";
+ return false;
+ }
+ return true;
+ }
+
+ private static function findMachinesForSubnet(&$subnet)
+ {
+ if (isset($subnet['dclients']))
+ return;
+ $cutoff = time() - 302;
+ // Get clients from same subnet first
+ $subnet['dclients'] = Database::queryAll("SELECT machineuuid, clientip FROM machine
+ WHERE state IN ('IDLE', 'OCCUPIED') AND INET_ATON(clientip) BETWEEN :start AND :end AND lastseen > :cutoff
+ LIMIT 3",
+ ['start' => $subnet['start'], 'end' => $subnet['end'], 'cutoff' => $cutoff]);
+ $subnet['iclients'] = [];
+ if (!empty($subnet['dclients']))
+ return;
+ // If none, get clients from other subnets known to be able to reach this one
+ $subnet['iclients'] = Database::queryAll("SELECT m.machineuuid, m.clientip FROM reboot_subnet_x_subnet sxs
+ INNER JOIN reboot_subnet s ON (s.subnetid = sxs.srcid AND sxs.dstid = :subnetid)
+ INNER JOIN machine m ON (INET_ATON(m.clientip) BETWEEN s.start AND s.end AND state IN ('IDLE', 'OCCUPIED') AND m.lastseen > :cutoff)
+ LIMIT 20", ['subnetid' => $subnet['subnetid'], 'cutoff' => $cutoff]);
+ shuffle($subnet['iclients']);
+ $subnet['iclients'] = array_slice($subnet['iclients'], 0, 3);
+ }
+
+}
generated by cgit v1.2.3-55-g7522 (git 2.39.0) at 2024-05-02 05:16:05 +0200