diff options
Diffstat (limited to 'modules-available/rebootcontrol/inc/rebootcontrol.inc.php')
| -rw-r--r-- | modules-available/rebootcontrol/inc/rebootcontrol.inc.php | 519 |
1 files changed, 490 insertions, 29 deletions
diff --git a/modules-available/rebootcontrol/inc/rebootcontrol.inc.php b/modules-available/rebootcontrol/inc/rebootcontrol.inc.php index ec4b84ed..107c2a50 100644 --- a/modules-available/rebootcontrol/inc/rebootcontrol.inc.php +++ b/modules-available/rebootcontrol/inc/rebootcontrol.inc.php @@ -5,81 +5,542 @@ class RebootControl const KEY_TASKLIST = 'rebootcontrol.tasklist'; + const KEY_AUTOSCAN_DISABLED = 'rebootcontrol.disable.scan'; + + const KEY_SCAN_CLIENT_TO_CLIENT = 'rebootcontrol.scan.c2c'; + + const KEY_UDP_PORT = 'rebootcontrol.port'; + + const KEY_BROADCAST_ADDRESS = 'rebootcontrol.broadcast-addr'; + const REBOOT = 'REBOOT'; const KEXEC_REBOOT = 'KEXEC_REBOOT'; const SHUTDOWN = 'SHUTDOWN'; + const TASK_REBOOTCTL = 'TASK_REBOOTCTL'; + const TASK_WOL = 'WAKE_ON_LAN'; + const TASK_EXEC = 'REMOTE_EXEC'; /** * @param string[] $uuids List of machineuuids to reboot * @param bool $kexec whether to trigger kexec-reboot instead of full BIOS cycle * @return false|array task struct for the reboot job */ - public static function reboot($uuids, $kexec = false) + public static function reboot(array $uuids, bool $kexec = false) { - $list = RebootQueries::getMachinesByUuid($uuids); + $list = RebootUtils::getMachinesByUuid($uuids); if (empty($list)) return false; - return self::execute($list, $kexec ? RebootControl::KEXEC_REBOOT : RebootControl::REBOOT, 0, 0); + return self::execute($list, $kexec ? RebootControl::KEXEC_REBOOT : RebootControl::REBOOT, 0); } /** - * @param array $list list of clients containing each keys 'machineuuid' and 'clientip' + * @param array $list list of clients containing each keys 'machineuuid', 'clientip' and 'locationid' * @param string $mode reboot mode: RebootControl::REBOOT ::KEXEC_REBOOT or ::SHUTDOWN * @param int $minutes delay in minutes for action - * @param int $locationId meta data only: locationId of clients * @return array|false the task, or false if it could not be started */ - public static function execute($list, $mode, $minutes, $locationId) + public static function execute(array $list, string $mode, int $minutes) { $task = Taskmanager::submit("RemoteReboot", array( "clients" => $list, "mode" => $mode, "minutes" => $minutes, - "locationId" => $locationId, "sshkey" => SSHKey::getPrivateKey(), "port" => 9922, // Hard-coded, must match mgmt-sshd module )); if (!Taskmanager::isFailed($task)) { - Property::addToList(RebootControl::KEY_TASKLIST, $locationId . '/' . $task["id"], 60 * 24); + self::addTask($task['id'], self::TASK_REBOOTCTL, $list, ['action' => $mode]); + foreach ($list as $client) { + $client['mode'] = $mode; + $client['minutes'] = $minutes; + EventLog::applyFilterRules('#action-power', $client); + } } return $task; } /** + * Add wake task metadata to database, so we can display job details on the summary page. + */ + private static function addTask(string $taskId, string $type, array $clients, array $other = null): void + { + $lids = ArrayUtil::flattenByKey($clients, 'locationid'); + $lids = array_unique($lids); + $newClients = []; + foreach ($clients as $c) { + $d = ['clientip' => $c['clientip']]; + if (isset($c['machineuuid'])) { + $d['machineuuid'] = $c['machineuuid']; + } + $newClients[] = $d; + } + $data = [ + 'id' => $taskId, + 'type' => $type, + 'locations' => $lids, + 'clients' => $newClients, + 'tasks' => [$taskId], // This did hold multiple tasks in the past; keep it in case we need this again + 'timestamp' => time(), + ]; + if (is_array($other)) { + $data += $other; + } + Property::addToList(RebootControl::KEY_TASKLIST, json_encode($data), 20); + } + + /** * @param int[]|null $locations filter by these locations - * @return array list of active tasks for reboots/shutdowns. + * @param ?string $id only with this TaskID + * @return array|false list of active tasks for reboots/shutdowns. */ - public static function getActiveTasks($locations = null) + public static function getActiveTasks(array $locations = null, string $id = null) { - if (is_array($locations) && in_array(0,$locations)) { + if (is_array($locations) && in_array(0, $locations)) { $locations = null; } $list = Property::getList(RebootControl::KEY_TASKLIST); $return = []; - foreach ($list as $entry) { - $p = explode('/', $entry, 2); - if (count($p) !== 2) { - Property::removeFromList(RebootControl::KEY_TASKLIST, $entry); + foreach ($list as $subkey => $entry) { + $p = json_decode($entry, true); + if (!is_array($p) || !isset($p['id'])) { + Property::removeFromListByKey(RebootControl::KEY_TASKLIST, $subkey); continue; } - if (is_array($locations) && !in_array($p[0], $locations)) // Ignore + if (is_array($locations) && is_array($p['locations']) && array_diff($p['locations'], $locations) !== []) + continue; // Not allowed + if ($id !== null) { + if ($p['id'] === $id) + return $p; continue; - $id = $p[1]; - $task = Taskmanager::status($id); - if (!Taskmanager::isTask($task)) { - Property::removeFromList(RebootControl::KEY_TASKLIST, $entry); + } + $valid = empty($p['tasks']); + if (!$valid) { + // Validate at least one task is still valid + foreach ($p['tasks'] as $task) { + $task = Taskmanager::status($task); + if (Taskmanager::isTask($task)) { + $p['status'] = $task['statusCode']; + $valid = true; + break; + } + } + } + if (!$valid) { + Property::removeFromListByKey(RebootControl::KEY_TASKLIST, $subkey); continue; } - $return[] = [ - 'taskId' => $task['id'], - 'locationId' => $task['data']['locationId'], - 'time' => $task['data']['time'], - 'mode' => $task['data']['mode'], - 'clientCount' => count($task['data']['clients']), - 'status' => $task['statusCode'], - ]; + $return[] = $p; } + if ($id !== null) + return false; return $return; } -}
\ No newline at end of file + /** + * Execute given command or script on a list of hosts. The list of hosts is an array of structs containing + * each a known machine-uuid and/or hostname, and optionally a port to use, which would otherwise default to 9922, + * and optionally a username to use, which would default to root. + * The command should be compatible with the remote user's default shell (most likely bash). + * + * @param array $clients [ { clientip: <host>, machineuuid: <uuid>, port: <port>, username: <username> }, ... ] + * @param string $command Command or script to execute on client + * @param int $timeout in seconds + * @param string|false $privkey SSH private key to use to connect + * @return array|false task struct, false on error + */ + public static function runScript(array $clients, string $command, int $timeout = 5, $privkey = false) + { + $task = self::runScriptInternal($clients, $command, $timeout, $privkey); + if (!Taskmanager::isFailed($task)) { + self::addTask($task['id'], self::TASK_EXEC, $clients); + } + return $task; + } + + private static function runScriptInternal(array &$clients, string $command, int $timeout = 5, $privkey = false) + { + $valid = []; + $invalid = []; + foreach ($clients as $client) { + if (is_string($client)) { + $invalid[strtoupper($client)] = []; // Assume machineuuid + } elseif (!isset($client['clientip']) && !isset($client['machineuuid'])) { + error_log('RebootControl::runScript called with list entry that has neither IP nor UUID'); + } elseif (!isset($client['clientip'])) { + $invalid[$client['machineuuid']] = $client; + } else { + $valid[] = $client; + } + } + if (!empty($invalid)) { + $res = Database::simpleQuery('SELECT machineuuid, clientip, locationid FROM machine WHERE machineuuid IN (:uuids)', + ['uuids' => array_keys($invalid)]); + foreach ($res as $row) { + if (isset($invalid[$row['machineuuid']])) { + $valid[] = $row + $invalid[$row['machineuuid']]; + } else { + $valid[] = $row; + } + } + } + $clients = $valid; + if (empty($clients)) { + error_log('RebootControl::runScript called without any clients'); + return false; + } + if ($privkey === false) { + $privkey = SSHKey::getPrivateKey(); + } + return Taskmanager::submit('RemoteExec', [ + 'clients' => $clients, + 'command' => $command, + 'timeoutSeconds' => $timeout, + 'sshkey' => $privkey, + 'port' => 9922, // Fallback if no port given in client struct + ]); + } + + public static function connectionCheckCallback($task, $hostId) + { + $reachable = 0; + if (isset($task['data']['result'])) { + foreach ($task['data']['result'] as $res) { + if ($res['exitCode'] == 0) { + $reachable = 1; + } + } + } + Database::exec('UPDATE reboot_jumphost SET reachable = :reachable WHERE hostid = :id', + ['id' => $hostId, 'reachable' => $reachable]); + } + + /** + * Wake clients given by MAC address(es) via jawol util. + * Multiple MAC addresses can be passed as a space separated list. + */ + private static function buildClientWakeCommand(string $macs, string $bcast = null, string $passwd = null): string + { + $command = 'jawol'; + if (!empty($bcast)) { + $command .= " -d '$bcast'"; + } else { + $command .= ' -i br0'; + } + if (!empty($passwd)) { + $command .= " -p '$passwd'"; + } + $command .= " $macs"; + return $command; + } + + /** + * @param array $sourceMachines list of source machines. array of [clientip, machineuuid] entries + * @param string $macaddr destination mac address(es) + * @param string $bcast directed broadcast address to send to + * @param string $passwd optional WOL password, mac address or ipv4 notation + * @return array|false task struct, false on error + */ + public static function wakeViaClient(array $sourceMachines, string $macaddr, string $bcast = null, string $passwd = null) + { + $command = self::buildClientWakeCommand($macaddr, $bcast, $passwd); + // Yes there is one zero "missing" from the usleep -- that's the whole point: we prefer 100ms sleeps + return self::runScriptInternal($sourceMachines, + "for i in 1 1 0; do $command; usleep \${i}00000 2> /dev/null || sleep \$i; done"); + } + + /** + * @param string|string[] $macaddr destination mac address(es) + * @param ?string $bcast directed broadcast address to send to + * @param ?string $passwd optional WOL password; mac address or ipv4 notation + * @return array|false task struct, false on error + */ + public static function wakeDirectly($macaddr, string $bcast = null, string $passwd = null) + { + if (!is_array($macaddr)) { + $macaddr = [$macaddr]; + } + $port = (int)Property::get(RebootControl::KEY_UDP_PORT); + if ($port < 1 || $port > 65535) { + $port = 9; + } + $arg = []; + foreach ($macaddr as $mac) { + $arg[] = [ + 'ip' => $bcast, + 'mac' => $mac, + 'methods' => ['DIRECT'], + 'password' => $passwd, + ]; + } + return Taskmanager::submit('WakeOnLan', ['clients' => $arg]); + } + + /** + * Explicitly wake given clients via jumphost + * @param array $jumphost the according row from the database, representing the desired jumphost + * @param string $bcast (directed) broadcast address for WOL packet, %IP% in command template + * @param array $clients list of clients, must contain at least key 'macaddr' for every client + * @return array|false task struct on successful submission to TM, false on error + */ + public static function wakeViaJumpHost(array $jumphost, string $bcast, array $clients) + { + $hostid = $jumphost['hostid']; + $macs = ArrayUtil::flattenByKey($clients, 'macaddr'); + if (empty($macs)) { + error_log('Called wakeViaJumpHost without clients'); + return false; + } + $macs = "'" . implode("' '", $macs) . "'"; + $macs = str_replace('-', ':', $macs); + $script = str_replace(['%IP%', '%MACS%'], [$bcast, $macs], $jumphost['script']); + $arg = [[ + 'clientip' => $jumphost['host'], + 'port' => $jumphost['port'], + 'username' => $jumphost['username'], + ]]; + $task = RebootControl::runScriptInternal($arg, $script, 6, $jumphost['sshkey']); + if ($task !== false && isset($task['id'])) { + TaskmanagerCallback::addCallback($task, 'rbcConnCheck', $hostid); + } + return $task; + } + + /** + * @param array $clientList list of clients containing each keys 'macaddr' and 'clientip', optionally 'locationid' + * @param ?array $failed list of failed clients from $clientList + * @return ?string taskid of this job + */ + public static function wakeMachines(array $clientList, array &$failed = null): ?string + { + $errors = ''; + $sent = $unknown = $unreachable = $failed = []; + // For event filtering by rule + // Need all subnets... + /* subnetid => [ + * subnetid => 1234, + * start => 1234, (ip2long) + * end => 5678, (ip2long) + * jumphosts => [id1, id2, ...], + */ + $subnets = []; + $res = Database::simpleQuery('SELECT subnetid, start, end, isdirect FROM reboot_subnet'); + foreach ($res as $row) { + $row += [ + 'djumphosts' => [], + 'ijumphosts' => [], + ]; + $subnets[$row['subnetid']] = $row; + } + // Get all jump hosts + self::addJumphostsToSubnets($subnets); + // Determine method for all clients + $taskClients = []; // array of arrays with keys [ip, mac, methods] + $taskSsh = []; // SSH configs for task, array of arrays with keys [username, sshkey, ip, port, command] + $overrideBroadcast = Property::get(self::KEY_BROADCAST_ADDRESS); + if (empty($overrideBroadcast)) { + $overrideBroadcast = false; + } + foreach ($clientList as $dbClient) { + $ip = sprintf('%u', ip2long($dbClient['clientip'])); // 32Bit snprintf + unset($subnet); + $subnet = false; + foreach ($subnets as &$sn) { + if ($sn['start'] <= $ip && $sn['end'] >= $ip) { + $subnet =& $sn; + break; + } + } + if ($subnet === false) { + $unknown[] = $dbClient; + continue; + } + $taskClient = [ + 'ip' => long2ip($subnet['end']), + 'mac' => $dbClient['macaddr'], + 'methods' => [], + ]; + // If we have an override broadcast address, unconditionally add this as the + // first method + if ($overrideBroadcast !== false) { + $taskClient['ip'] = $overrideBroadcast; + $taskClient['methods'][] = 'DIRECT'; + } + self::findMachinesForSubnet($subnet); + // Highest priority - clients in same subnet, no directed broadcast + // required, should be most reliable + self::addSshMethodUsingClient($subnet['dclients'], $taskClient['methods'], $taskSsh); + // Jumphost - usually in same subnet + self::addSshMethodUsingJumphost($subnet['djumphosts'], true, $taskClient['methods'], $taskSsh); + // Jumphosts in other subnets, determined to be able to reach destination subnet + self::addSshMethodUsingJumphost($subnet['ijumphosts'], true, $taskClient['methods'], $taskSsh); + // If directly reachable from server, prefer this now over the questionable approaches below, + // but only if we didn't already add this above because of override + if ($overrideBroadcast === false && $subnet['isdirect']) { + $taskClient['methods'][] = 'DIRECT'; + } + // Use clients in other subnets, known to be able to reach the destination net + self::addSshMethodUsingClient($subnet['iclients'], $taskClient['methods'], $taskSsh); + // Add warning if nothing works + if (empty($taskClient['methods'])) { + $unreachable[] = $dbClient; + } else { + // TODO: Remember WOL was attempted + } + // "Questionable approaches": + // Last fallback is jumphosts that were not reachable when last checked, this is really a last resort + self::addSshMethodUsingJumphost($subnet['djumphosts'], false, $taskClient['methods'], $taskSsh); + self::addSshMethodUsingJumphost($subnet['ijumphosts'], false, $taskClient['methods'], $taskSsh); + + if (!empty($taskClient['methods'])) { + $taskClients[] = $taskClient; + $sent[] = $dbClient; + } + } + unset($subnet); + + if (!empty($unknown)) { + $ips = ArrayUtil::flattenByKey($unknown, 'clientip'); + $errors .= "**** WARNING ****\nThe following clients do not belong to a known subnet (bug?)\n" . implode("\n", $ips) . "\n"; + foreach ($unknown as $val) { + $failed[$val['clientip']] = $val; + } + } + if (!empty($unreachable)) { + $ips = ArrayUtil::flattenByKey($unreachable, 'clientip'); + $errors .= "**** WARNING ****\nThe following clients are not reachable with any method\n" . implode("\n", $ips) . "\n"; + foreach ($unreachable as $val) { + $failed[$val['clientip']] = $val; + } + } + $task = Taskmanager::submit('WakeOnLan', [ + 'clients' => $taskClients, + 'ssh' => $taskSsh, + ]); + if (isset($task['id'])) { + $id = $task['id']; + self::addTask($id, self::TASK_WOL, $clientList, ['log' => $errors]); + foreach ($sent as $dbClient) { + EventLog::applyFilterRules('#action-wol', $dbClient); + } + return $id; + } + return null; + } + + private static function findMachinesForSubnet(&$subnet) + { + if (isset($subnet['dclients'])) + return; + $cutoff = time() - 320; + // Get clients from same subnet first + $subnet['dclients'] = Database::queryColumnArray("SELECT clientip FROM machine + WHERE state IN ('IDLE', 'OCCUPIED') AND INET_ATON(clientip) BETWEEN :start AND :end AND lastseen > :cutoff + LIMIT 3", + ['start' => $subnet['start'], 'end' => $subnet['end'], 'cutoff' => $cutoff]); + // If none, get clients from other subnets known to be able to reach this one + $subnet['iclients'] = Database::queryColumnArray("SELECT m.clientip FROM reboot_subnet_x_subnet sxs + INNER JOIN reboot_subnet s ON (s.subnetid = sxs.srcid AND sxs.dstid = :subnetid AND sxs.reachable = 1) + INNER JOIN machine m ON (INET_ATON(m.clientip) BETWEEN s.start AND s.end AND state IN ('IDLE', 'OCCUPIED') AND m.lastseen > :cutoff) + LIMIT 20", ['subnetid' => $subnet['subnetid'], 'cutoff' => $cutoff]); + shuffle($subnet['iclients']); + $subnet['iclients'] = array_slice($subnet['iclients'], 0, 3); + } + + public static function prepareExec() + { + User::assertPermission('.rebootcontrol.action.exec'); + $uuids = array_values(Request::post('uuid', Request::REQUIRED, 'array')); + $machines = RebootUtils::getFilteredMachineList($uuids, '.rebootcontrol.action.exec'); + if ($machines === false) + return; + $id = mt_rand(); + Session::set('exec-' . $id, $machines, 60); + Util::redirect('?do=rebootcontrol&show=exec&what=prepare&id=' . $id); + } + + /** + * Append a "wake via client" WOL method for the given client. Append at least one, but stop + * if there are at least two methods already. + * + * @param array $sshClients [in] list of online clients to use for waking + * @param array $c [out] The client's methods array + * @param array $taskSsh [out] add according task struct to this array, if not already exists + * @return void + */ + private static function addSshMethodUsingClient(array $sshClients, array &$methods, array &$taskSsh) + { + foreach ($sshClients as $host) { + if (!isset($taskSsh[$host])) { + $taskSsh[$host] = [ + 'username' => 'root', + 'sshkey' => SSHKey::getPrivateKey(), + 'ip' => $host, + 'port' => 9922, + 'command' => self::buildClientWakeCommand('%MACS%', '%IP%'), + ]; + } + $methods[] = $host; + if (count($methods) >= 2) + break; + } + } + + private static function addSshMethodUsingJumphost(array $jumpHosts, bool $reachable, array &$methods, array &$taskSsh) + { + // If it's the fallback to apparently unreachable jump-hosts, ignore if we already have two methods + if (!$reachable && count($methods) >= 2) + return; + // host, port, username, sshkey, script, jh.reachable + foreach ($jumpHosts as $jh) { + if ($reachable !== (bool)$jh['reachable']) + continue; + $key = substr(md5($jh['host'] . ':' . $jh['port'] . ':' . $jh['username']), 0, 10); + if (!isset($taskSsh[$key])) { + $taskSsh[$key] = [ + 'username' => $jh['username'], + 'sshkey' => $jh['sshkey'], + 'ip' => $jh['host'], + 'port' => $jh['port'], + 'command' => $jh['script'], + ]; + } + $methods[] = $key; + if (count($methods) >= 2) + break; + } + } + + /** + * Load all jumphosts from DB, sort into passed $subnets. Also split up + * by directly assigned subnets, and indirectly dtermined, reachable subnets. + * @param array $subnets [in] + * @return void + */ + private static function addJumphostsToSubnets(array &$subnets) + { + $res = Database::simpleQuery('SELECT host, port, username, sshkey, script, jh.reachable, + Group_Concat(jxs.subnetid) AS dsubnets, Group_Concat(sxs.dstid) AS isubnets + FROM reboot_jumphost jh + LEFT JOIN reboot_jumphost_x_subnet jxs ON (jh.hostid = jxs.hostid) + LEFT JOIN reboot_subnet s ON (INET_ATON(jh.host) BETWEEN s.start AND s.end) + LEFT JOIN reboot_subnet_x_subnet sxs ON (sxs.srcid = s.subnetid AND sxs.reachable <> 0) + GROUP BY jh.hostid'); + foreach ($res as $row) { + $dnets = empty($row['dsubnets']) ? [] : explode(',', $row['dsubnets']); + $inets = empty($row['isubnets']) ? [] : explode(',', $row['isubnets']); + $inets = array_diff($inets, $dnets); // There might be duplicates if both joins match + foreach ($dnets as $net) { + if (empty($net) || !isset($subnets[$net])) + continue; + $subnets[$net]['djumphosts'][] =& $row; + } + foreach ($inets as $net) { + if (empty($net) || !isset($subnets[$net])) + continue; + $subnets[$net]['ijumphosts'][] =& $row; + } + unset($row); + } + } + +} |