<?php
class RebootControl
{
const KEY_TASKLIST = 'rebootcontrol.tasklist';
const KEY_AUTOSCAN_DISABLED = 'rebootcontrol.disable.scan';
const KEY_SCAN_CLIENT_TO_CLIENT = 'rebootcontrol.scan.c2c';
const KEY_UDP_PORT = 'rebootcontrol.port';
const KEY_BROADCAST_ADDRESS = 'rebootcontrol.broadcast-addr';
const REBOOT = 'REBOOT';
const KEXEC_REBOOT = 'KEXEC_REBOOT';
const SHUTDOWN = 'SHUTDOWN';
const TASK_REBOOTCTL = 'TASK_REBOOTCTL';
const TASK_WOL = 'WAKE_ON_LAN';
const TASK_EXEC = 'REMOTE_EXEC';
/**
* @param string[] $uuids List of machineuuids to reboot
* @param bool $kexec whether to trigger kexec-reboot instead of full BIOS cycle
* @return false|array task struct for the reboot job
*/
public static function reboot(array $uuids, bool $kexec = false)
{
$list = RebootUtils::getMachinesByUuid($uuids);
if (empty($list))
return false;
return self::execute($list, $kexec ? RebootControl::KEXEC_REBOOT : RebootControl::REBOOT, 0);
}
/**
* @param array $list list of clients containing each keys 'machineuuid', 'clientip' and 'locationid'
* @param string $mode reboot mode: RebootControl::REBOOT ::KEXEC_REBOOT or ::SHUTDOWN
* @param int $minutes delay in minutes for action
* @return array|false the task, or false if it could not be started
*/
public static function execute(array $list, string $mode, int $minutes)
{
$task = Taskmanager::submit("RemoteReboot", array(
"clients" => $list,
"mode" => $mode,
"minutes" => $minutes,
"sshkey" => SSHKey::getPrivateKey(),
"port" => 9922, // Hard-coded, must match mgmt-sshd module
));
if (!Taskmanager::isFailed($task)) {
self::addTask($task['id'], self::TASK_REBOOTCTL, $list, ['action' => $mode]);
foreach ($list as $client) {
$client['mode'] = $mode;
$client['minutes'] = $minutes;
EventLog::applyFilterRules('#action-power', $client);
}
}
return $task;
}
/**
* Add wake task metadata to database, so we can display job details on the summary page.
*/
private static function addTask(string $taskId, string $type, array $clients, array $other = null): void
{
$lids = ArrayUtil::flattenByKey($clients, 'locationid');
$lids = array_unique($lids);
$newClients = [];
foreach ($clients as $c) {
$d = ['clientip' => $c['clientip']];
if (isset($c['machineuuid'])) {
$d['machineuuid'] = $c['machineuuid'];
}
$newClients[] = $d;
}
$data = [
'id' => $taskId,
'type' => $type,
'locations' => $lids,
'clients' => $newClients,
'tasks' => [$taskId], // This did hold multiple tasks in the past; keep it in case we need this again
'timestamp' => time(),
];
if (is_array($other)) {
$data += $other;
}
Property::addToList(RebootControl::KEY_TASKLIST, json_encode($data), 20);
}
/**
* @param int[]|null $locations filter by these locations
* @param ?string $id only with this TaskID
* @return array|false list of active tasks for reboots/shutdowns.
*/
public static function getActiveTasks(array $locations = null, string $id = null)
{
if (is_array($locations) && in_array(0, $locations)) {
$locations = null;
}
$list = Property::getList(RebootControl::KEY_TASKLIST);
$return = [];
foreach ($list as $subkey => $entry) {
$p = json_decode($entry, true);
if (!is_array($p) || !isset($p['id'])) {
Property::removeFromListByKey(RebootControl::KEY_TASKLIST, $subkey);
continue;
}
if (is_array($locations) && is_array($p['locations']) && array_diff($p['locations'], $locations) !== [])
continue; // Not allowed
if ($id !== null) {
if ($p['id'] === $id)
return $p;
continue;
}
$valid = empty($p['tasks']);
if (!$valid) {
// Validate at least one task is still valid
foreach ($p['tasks'] as $task) {
$task = Taskmanager::status($task);
if (Taskmanager::isTask($task)) {
$p['status'] = $task['statusCode'];
$valid = true;
break;
}
}
}
if (!$valid) {
Property::removeFromListByKey(RebootControl::KEY_TASKLIST, $subkey);
continue;
}
$return[] = $p;
}
if ($id !== null)
return false;
return $return;
}
/**
* Execute given command or script on a list of hosts. The list of hosts is an array of structs containing
* each a known machine-uuid and/or hostname, and optionally a port to use, which would otherwise default to 9922,
* and optionally a username to use, which would default to root.
* The command should be compatible with the remote user's default shell (most likely bash).
*
* @param array $clients [ { clientip: <host>, machineuuid: <uuid>, port: <port>, username: <username> }, ... ]
* @param string $command Command or script to execute on client
* @param int $timeout in seconds
* @param string|false $privkey SSH private key to use to connect
* @return array|false task struct, false on error
*/
public static function runScript(array $clients, string $command, int $timeout = 5, $privkey = false)
{
$task = self::runScriptInternal($clients, $command, $timeout, $privkey);
if (!Taskmanager::isFailed($task)) {
self::addTask($task['id'], self::TASK_EXEC, $clients);
}
return $task;
}
private static function runScriptInternal(array &$clients, string $command, int $timeout = 5, $privkey = false)
{
$valid = [];
$invalid = [];
foreach ($clients as $client) {
if (is_string($client)) {
$invalid[strtoupper($client)] = []; // Assume machineuuid
} elseif (!isset($client['clientip']) && !isset($client['machineuuid'])) {
error_log('RebootControl::runScript called with list entry that has neither IP nor UUID');
} elseif (!isset($client['clientip'])) {
$invalid[$client['machineuuid']] = $client;
} else {
$valid[] = $client;
}
}
if (!empty($invalid)) {
$res = Database::simpleQuery('SELECT machineuuid, clientip, locationid FROM machine WHERE machineuuid IN (:uuids)',
['uuids' => array_keys($invalid)]);
foreach ($res as $row) {
if (isset($invalid[$row['machineuuid']])) {
$valid[] = $row + $invalid[$row['machineuuid']];
} else {
$valid[] = $row;
}
}
}
$clients = $valid;
if (empty($clients)) {
error_log('RebootControl::runScript called without any clients');
return false;
}
if ($privkey === false) {
$privkey = SSHKey::getPrivateKey();
}
return Taskmanager::submit('RemoteExec', [
'clients' => $clients,
'command' => $command,
'timeoutSeconds' => $timeout,
'sshkey' => $privkey,
'port' => 9922, // Fallback if no port given in client struct
]);
}
public static function connectionCheckCallback($task, $hostId)
{
$reachable = 0;
if (isset($task['data']['result'])) {
foreach ($task['data']['result'] as $res) {
if ($res['exitCode'] == 0) {
$reachable = 1;
}
}
}
Database::exec('UPDATE reboot_jumphost SET reachable = :reachable WHERE hostid = :id',
['id' => $hostId, 'reachable' => $reachable]);
}
/**
* Wake clients given by MAC address(es) via jawol util.
* Multiple MAC addresses can be passed as a space separated list.
*/
private static function buildClientWakeCommand(string $macs, string $bcast = null, string $passwd = null): string
{
$command = 'jawol';
if (!empty($bcast)) {
$command .= " -d '$bcast'";
} else {
$command .= ' -i br0';
}
if (!empty($passwd)) {
$command .= " -p '$passwd'";
}
$command .= " $macs";
return $command;
}
/**
* @param array $sourceMachines list of source machines. array of [clientip, machineuuid] entries
* @param string $macaddr destination mac address(es)
* @param string $bcast directed broadcast address to send to
* @param string $passwd optional WOL password, mac address or ipv4 notation
* @return array|false task struct, false on error
*/
public static function wakeViaClient(array $sourceMachines, string $macaddr, string $bcast = null, string $passwd = null)
{
$command = self::buildClientWakeCommand($macaddr, $bcast, $passwd);
// Yes there is one zero "missing" from the usleep -- that's the whole point: we prefer 100ms sleeps
return self::runScriptInternal($sourceMachines,
"for i in 1 1 0; do $command; usleep \${i}00000 2> /dev/null || sleep \$i; done");
}
/**
* @param string|string[] $macaddr destination mac address(es)
* @param ?string $bcast directed broadcast address to send to
* @param ?string $passwd optional WOL password; mac address or ipv4 notation
* @return array|false task struct, false on error
*/
public static function wakeDirectly($macaddr, string $bcast = null, string $passwd = null)
{
if (!is_array($macaddr)) {
$macaddr = [$macaddr];
}
$port = (int)Property::get(RebootControl::KEY_UDP_PORT);
if ($port < 1 || $port > 65535) {
$port = 9;
}
$arg = [];
foreach ($macaddr as $mac) {
$arg[] = [
'ip' => $bcast,
'mac' => $mac,
'methods' => ['DIRECT'],
'password' => $passwd,
];
}
return Taskmanager::submit('WakeOnLan', ['clients' => $arg]);
}
/**
* Explicitly wake given clients via jumphost
* @param array $jumphost the according row from the database, representing the desired jumphost
* @param string $bcast (directed) broadcast address for WOL packet, %IP% in command template
* @param array $clients list of clients, must contain at least key 'macaddr' for every client
* @return array|false task struct on successful submission to TM, false on error
*/
public static function wakeViaJumpHost(array $jumphost, string $bcast, array $clients)
{
$hostid = $jumphost['hostid'];
$macs = ArrayUtil::flattenByKey($clients, 'macaddr');
if (empty($macs)) {
error_log('Called wakeViaJumpHost without clients');
return false;
}
$macs = "'" . implode("' '", $macs) . "'";
$macs = str_replace('-', ':', $macs);
$script = str_replace(['%IP%', '%MACS%'], [$bcast, $macs], $jumphost['script']);
$arg = [[
'clientip' => $jumphost['host'],
'port' => $jumphost['port'],
'username' => $jumphost['username'],
]];
$task = RebootControl::runScriptInternal($arg, $script, 6, $jumphost['sshkey']);
if ($task !== false && isset($task['id'])) {
TaskmanagerCallback::addCallback($task, 'rbcConnCheck', $hostid);
}
return $task;
}
/**
* @param array $clientList list of clients containing each keys 'macaddr' and 'clientip', optionally 'locationid'
* @param ?array $failed list of failed clients from $clientList
* @return ?string taskid of this job
*/
public static function wakeMachines(array $clientList, array &$failed = null): ?string
{
$errors = '';
$sent = $unknown = $unreachable = $failed = [];
// For event filtering by rule
// Need all subnets...
/* subnetid => [
* subnetid => 1234,
* start => 1234, (ip2long)
* end => 5678, (ip2long)
* jumphosts => [id1, id2, ...],
*/
$subnets = [];
$res = Database::simpleQuery('SELECT subnetid, start, end, isdirect FROM reboot_subnet');
foreach ($res as $row) {
$row += [
'djumphosts' => [],
'ijumphosts' => [],
];
$subnets[$row['subnetid']] = $row;
}
// Get all jump hosts
self::addJumphostsToSubnets($subnets);
// Determine method for all clients
$taskClients = []; // array of arrays with keys [ip, mac, methods]
$taskSsh = []; // SSH configs for task, array of arrays with keys [username, sshkey, ip, port, command]
$overrideBroadcast = Property::get(self::KEY_BROADCAST_ADDRESS);
if (empty($overrideBroadcast)) {
$overrideBroadcast = false;
}
foreach ($clientList as $dbClient) {
$ip = sprintf('%u', ip2long($dbClient['clientip'])); // 32Bit snprintf
unset($subnet);
$subnet = false;
foreach ($subnets as &$sn) {
if ($sn['start'] <= $ip && $sn['end'] >= $ip) {
$subnet =& $sn;
break;
}
}
if ($subnet === false) {
$unknown[] = $dbClient;
continue;
}
$taskClient = [
'ip' => long2ip($subnet['end']),
'mac' => $dbClient['macaddr'],
'methods' => [],
];
// If we have an override broadcast address, unconditionally add this as the
// first method
if ($overrideBroadcast !== false) {
$taskClient['ip'] = $overrideBroadcast;
$taskClient['methods'][] = 'DIRECT';
}
self::findMachinesForSubnet($subnet);
// Highest priority - clients in same subnet, no directed broadcast
// required, should be most reliable
self::addSshMethodUsingClient($subnet['dclients'], $taskClient['methods'], $taskSsh);
// Jumphost - usually in same subnet
self::addSshMethodUsingJumphost($subnet['djumphosts'], true, $taskClient['methods'], $taskSsh);
// Jumphosts in other subnets, determined to be able to reach destination subnet
self::addSshMethodUsingJumphost($subnet['ijumphosts'], true, $taskClient['methods'], $taskSsh);
// If directly reachable from server, prefer this now over the questionable approaches below,
// but only if we didn't already add this above because of override
if ($overrideBroadcast === false && $subnet['isdirect']) {
$taskClient['methods'][] = 'DIRECT';
}
// Use clients in other subnets, known to be able to reach the destination net
self::addSshMethodUsingClient($subnet['iclients'], $taskClient['methods'], $taskSsh);
// Add warning if nothing works
if (empty($taskClient['methods'])) {
$unreachable[] = $dbClient;
} else {
// TODO: Remember WOL was attempted
}
// "Questionable approaches":
// Last fallback is jumphosts that were not reachable when last checked, this is really a last resort
self::addSshMethodUsingJumphost($subnet['djumphosts'], false, $taskClient['methods'], $taskSsh);
self::addSshMethodUsingJumphost($subnet['ijumphosts'], false, $taskClient['methods'], $taskSsh);
if (!empty($taskClient['methods'])) {
$taskClients[] = $taskClient;
$sent[] = $dbClient;
}
}
unset($subnet);
if (!empty($unknown)) {
$ips = ArrayUtil::flattenByKey($unknown, 'clientip');
$errors .= "**** WARNING ****\nThe following clients do not belong to a known subnet (bug?)\n" . implode("\n", $ips) . "\n";
foreach ($unknown as $val) {
$failed[$val['clientip']] = $val;
}
}
if (!empty($unreachable)) {
$ips = ArrayUtil::flattenByKey($unreachable, 'clientip');
$errors .= "**** WARNING ****\nThe following clients are not reachable with any method\n" . implode("\n", $ips) . "\n";
foreach ($unreachable as $val) {
$failed[$val['clientip']] = $val;
}
}
$task = Taskmanager::submit('WakeOnLan', [
'clients' => $taskClients,
'ssh' => $taskSsh,
]);
if (isset($task['id'])) {
$id = $task['id'];
self::addTask($id, self::TASK_WOL, $clientList, ['log' => $errors]);
foreach ($sent as $dbClient) {
EventLog::applyFilterRules('#action-wol', $dbClient);
}
return $id;
}
return null;
}
private static function findMachinesForSubnet(&$subnet)
{
if (isset($subnet['dclients']))
return;
$cutoff = time() - 320;
// Get clients from same subnet first
$subnet['dclients'] = Database::queryColumnArray("SELECT clientip FROM machine
WHERE state IN ('IDLE', 'OCCUPIED') AND INET_ATON(clientip) BETWEEN :start AND :end AND lastseen > :cutoff
LIMIT 3",
['start' => $subnet['start'], 'end' => $subnet['end'], 'cutoff' => $cutoff]);
// If none, get clients from other subnets known to be able to reach this one
$subnet['iclients'] = Database::queryColumnArray("SELECT m.clientip FROM reboot_subnet_x_subnet sxs
INNER JOIN reboot_subnet s ON (s.subnetid = sxs.srcid AND sxs.dstid = :subnetid AND sxs.reachable = 1)
INNER JOIN machine m ON (INET_ATON(m.clientip) BETWEEN s.start AND s.end AND state IN ('IDLE', 'OCCUPIED') AND m.lastseen > :cutoff)
LIMIT 20", ['subnetid' => $subnet['subnetid'], 'cutoff' => $cutoff]);
shuffle($subnet['iclients']);
$subnet['iclients'] = array_slice($subnet['iclients'], 0, 3);
}
public static function prepareExec()
{
User::assertPermission('.rebootcontrol.action.exec');
$uuids = array_values(Request::post('uuid', Request::REQUIRED, 'array'));
$machines = RebootUtils::getFilteredMachineList($uuids, '.rebootcontrol.action.exec');
if ($machines === false)
return;
$id = mt_rand();
Session::set('exec-' . $id, $machines, 60);
Util::redirect('?do=rebootcontrol&show=exec&what=prepare&id=' . $id);
}
/**
* Append a "wake via client" WOL method for the given client. Append at least one, but stop
* if there are at least two methods already.
*
* @param array $sshClients [in] list of online clients to use for waking
* @param array $c [out] The client's methods array
* @param array $taskSsh [out] add according task struct to this array, if not already exists
* @return void
*/
private static function addSshMethodUsingClient(array $sshClients, array &$methods, array &$taskSsh)
{
foreach ($sshClients as $host) {
if (!isset($taskSsh[$host])) {
$taskSsh[$host] = [
'username' => 'root',
'sshkey' => SSHKey::getPrivateKey(),
'ip' => $host,
'port' => 9922,
'command' => self::buildClientWakeCommand('%MACS%', '%IP%'),
];
}
$methods[] = $host;
if (count($methods) >= 2)
break;
}
}
private static function addSshMethodUsingJumphost(array $jumpHosts, bool $reachable, array &$methods, array &$taskSsh)
{
// If it's the fallback to apparently unreachable jump-hosts, ignore if we already have two methods
if (!$reachable && count($methods) >= 2)
return;
// host, port, username, sshkey, script, jh.reachable
foreach ($jumpHosts as $jh) {
if ($reachable !== (bool)$jh['reachable'])
continue;
$key = substr(md5($jh['host'] . ':' . $jh['port'] . ':' . $jh['username']), 0, 10);
if (!isset($taskSsh[$key])) {
$taskSsh[$key] = [
'username' => $jh['username'],
'sshkey' => $jh['sshkey'],
'ip' => $jh['host'],
'port' => $jh['port'],
'command' => $jh['script'],
];
}
$methods[] = $key;
if (count($methods) >= 2)
break;
}
}
/**
* Load all jumphosts from DB, sort into passed $subnets. Also split up
* by directly assigned subnets, and indirectly dtermined, reachable subnets.
* @param array $subnets [in]
* @return void
*/
private static function addJumphostsToSubnets(array &$subnets)
{
$res = Database::simpleQuery('SELECT host, port, username, sshkey, script, jh.reachable,
Group_Concat(jxs.subnetid) AS dsubnets, Group_Concat(sxs.dstid) AS isubnets
FROM reboot_jumphost jh
LEFT JOIN reboot_jumphost_x_subnet jxs ON (jh.hostid = jxs.hostid)
LEFT JOIN reboot_subnet s ON (INET_ATON(jh.host) BETWEEN s.start AND s.end)
LEFT JOIN reboot_subnet_x_subnet sxs ON (sxs.srcid = s.subnetid AND sxs.reachable <> 0)
GROUP BY jh.hostid');
foreach ($res as $row) {
$dnets = empty($row['dsubnets']) ? [] : explode(',', $row['dsubnets']);
$inets = empty($row['isubnets']) ? [] : explode(',', $row['isubnets']);
$inets = array_diff($inets, $dnets); // There might be duplicates if both joins match
foreach ($dnets as $net) {
if (empty($net) || !isset($subnets[$net]))
continue;
$subnets[$net]['djumphosts'][] =& $row;
}
foreach ($inets as $net) {
if (empty($net) || !isset($subnets[$net]))
continue;
$subnets[$net]['ijumphosts'][] =& $row;
}
unset($row);
}
}
}
