$list, "mode" => $mode, "minutes" => $minutes, "sshkey" => SSHKey::getPrivateKey(), "port" => 9922, // Hard-coded, must match mgmt-sshd module )); if (!Taskmanager::isFailed($task)) { self::addTask($task['id'], self::TASK_REBOOTCTL, $list, ['action' => $mode]); foreach ($list as $client) { $client['mode'] = $mode; $client['minutes'] = $minutes; EventLog::applyFilterRules('#action-power', $client); } } return $task; } /** * Add wake task metadata to database, so we can display job details on the summary page. */ private static function addTask(string $taskId, string $type, array $clients, array $other = null): void { $lids = ArrayUtil::flattenByKey($clients, 'locationid'); $lids = array_unique($lids); $newClients = []; foreach ($clients as $c) { $d = ['clientip' => $c['clientip']]; if (isset($c['machineuuid'])) { $d['machineuuid'] = $c['machineuuid']; } $newClients[] = $d; } $data = [ 'id' => $taskId, 'type' => $type, 'locations' => $lids, 'clients' => $newClients, 'tasks' => [$taskId], // This did hold multiple tasks in the past; keep it in case we need this again 'timestamp' => time(), ]; if (is_array($other)) { $data += $other; } Property::addToList(RebootControl::KEY_TASKLIST, json_encode($data), 20); } /** * @param int[]|null $locations filter by these locations * @param ?string $id only with this TaskID * @return array|false list of active tasks for reboots/shutdowns. */ public static function getActiveTasks(array $locations = null, string $id = null) { if (is_array($locations) && in_array(0, $locations)) { $locations = null; } $list = Property::getList(RebootControl::KEY_TASKLIST); $return = []; foreach ($list as $subkey => $entry) { $p = json_decode($entry, true); if (!is_array($p) || !isset($p['id'])) { Property::removeFromListByKey(RebootControl::KEY_TASKLIST, $subkey); continue; } if (is_array($locations) && is_array($p['locations']) && array_diff($p['locations'], $locations) !== []) continue; // Not allowed if ($id !== null) { if ($p['id'] === $id) return $p; continue; } $valid = empty($p['tasks']); if (!$valid) { // Validate at least one task is still valid foreach ($p['tasks'] as $task) { $task = Taskmanager::status($task); if (Taskmanager::isTask($task)) { $p['status'] = $task['statusCode']; $valid = true; break; } } } if (!$valid) { Property::removeFromListByKey(RebootControl::KEY_TASKLIST, $subkey); continue; } $return[] = $p; } if ($id !== null) return false; return $return; } /** * Execute given command or script on a list of hosts. The list of hosts is an array of structs containing * each a known machine-uuid and/or hostname, and optionally a port to use, which would otherwise default to 9922, * and optionally a username to use, which would default to root. * The command should be compatible with the remote user's default shell (most likely bash). * * @param array $clients [ { clientip: , machineuuid: , port: , username: }, ... ] * @param string $command Command or script to execute on client * @param int $timeout in seconds * @param string|false $privkey SSH private key to use to connect * @return array|false task struct, false on error */ public static function runScript(array $clients, string $command, int $timeout = 5, $privkey = false) { $task = self::runScriptInternal($clients, $command, $timeout, $privkey); if (!Taskmanager::isFailed($task)) { self::addTask($task['id'], self::TASK_EXEC, $clients); } return $task; } private static function runScriptInternal(array &$clients, string $command, int $timeout = 5, $privkey = false) { $valid = []; $invalid = []; foreach ($clients as $client) { if (is_string($client)) { $invalid[strtoupper($client)] = []; // Assume machineuuid } elseif (!isset($client['clientip']) && !isset($client['machineuuid'])) { error_log('RebootControl::runScript called with list entry that has neither IP nor UUID'); } elseif (!isset($client['clientip'])) { $invalid[$client['machineuuid']] = $client; } else { $valid[] = $client; } } if (!empty($invalid)) { $res = Database::simpleQuery('SELECT machineuuid, clientip, locationid FROM machine WHERE machineuuid IN (:uuids)', ['uuids' => array_keys($invalid)]); foreach ($res as $row) { if (isset($invalid[$row['machineuuid']])) { $valid[] = $row + $invalid[$row['machineuuid']]; } else { $valid[] = $row; } } } $clients = $valid; if (empty($clients)) { error_log('RebootControl::runScript called without any clients'); return false; } if ($privkey === false) { $privkey = SSHKey::getPrivateKey(); } return Taskmanager::submit('RemoteExec', [ 'clients' => $clients, 'command' => $command, 'timeoutSeconds' => $timeout, 'sshkey' => $privkey, 'port' => 9922, // Fallback if no port given in client struct ]); } public static function connectionCheckCallback($task, $hostId) { $reachable = 0; if (isset($task['data']['result'])) { foreach ($task['data']['result'] as $res) { if ($res['exitCode'] == 0) { $reachable = 1; } } } Database::exec('UPDATE reboot_jumphost SET reachable = :reachable WHERE hostid = :id', ['id' => $hostId, 'reachable' => $reachable]); } /** * Wake clients given by MAC address(es) via jawol util. * Multiple MAC addresses can be passed as a space separated list. */ private static function buildClientWakeCommand(string $macs, string $bcast = null, string $passwd = null): string { $command = 'jawol'; if (!empty($bcast)) { $command .= " -d '$bcast'"; } else { $command .= ' -i br0'; } if (!empty($passwd)) { $command .= " -p '$passwd'"; } $command .= " $macs"; return $command; } /** * @param array $sourceMachines list of source machines. array of [clientip, machineuuid] entries * @param string $macaddr destination mac address(es) * @param string $bcast directed broadcast address to send to * @param string $passwd optional WOL password, mac address or ipv4 notation * @return array|false task struct, false on error */ public static function wakeViaClient(array $sourceMachines, string $macaddr, string $bcast = null, string $passwd = null) { $command = self::buildClientWakeCommand($macaddr, $bcast, $passwd); // Yes there is one zero "missing" from the usleep -- that's the whole point: we prefer 100ms sleeps return self::runScriptInternal($sourceMachines, "for i in 1 1 0; do $command; usleep \${i}00000 2> /dev/null || sleep \$i; done"); } /** * @param string|string[] $macaddr destination mac address(es) * @param ?string $bcast directed broadcast address to send to * @param ?string $passwd optional WOL password; mac address or ipv4 notation * @return array|false task struct, false on error */ public static function wakeDirectly($macaddr, string $bcast = null, string $passwd = null) { if (!is_array($macaddr)) { $macaddr = [$macaddr]; } $port = (int)Property::get(RebootControl::KEY_UDP_PORT); if ($port < 1 || $port > 65535) { $port = 9; } $arg = []; foreach ($macaddr as $mac) { $arg[] = [ 'ip' => $bcast, 'mac' => $mac, 'methods' => ['DIRECT'], 'password' => $passwd, ]; } return Taskmanager::submit('WakeOnLan', ['clients' => $arg]); } /** * Explicitly wake given clients via jumphost * @param array $jumphost the according row from the database, representing the desired jumphost * @param string $bcast (directed) broadcast address for WOL packet, %IP% in command template * @param array $clients list of clients, must contain at least key 'macaddr' for every client * @return array|false task struct on successful submission to TM, false on error */ public static function wakeViaJumpHost(array $jumphost, string $bcast, array $clients) { $hostid = $jumphost['hostid']; $macs = ArrayUtil::flattenByKey($clients, 'macaddr'); if (empty($macs)) { error_log('Called wakeViaJumpHost without clients'); return false; } $macs = "'" . implode("' '", $macs) . "'"; $macs = str_replace('-', ':', $macs); $script = str_replace(['%IP%', '%MACS%'], [$bcast, $macs], $jumphost['script']); $arg = [[ 'clientip' => $jumphost['host'], 'port' => $jumphost['port'], 'username' => $jumphost['username'], ]]; $task = RebootControl::runScriptInternal($arg, $script, 6, $jumphost['sshkey']); if ($task !== false && isset($task['id'])) { TaskmanagerCallback::addCallback($task, 'rbcConnCheck', $hostid); } return $task; } /** * @param array $clientList list of clients containing each keys 'macaddr' and 'clientip', optionally 'locationid' * @param ?array $failed list of failed clients from $clientList * @return ?string taskid of this job */ public static function wakeMachines(array $clientList, array &$failed = null): ?string { $errors = ''; $sent = $unknown = $unreachable = $failed = []; // For event filtering by rule // Need all subnets... /* subnetid => [ * subnetid => 1234, * start => 1234, (ip2long) * end => 5678, (ip2long) * jumphosts => [id1, id2, ...], */ $subnets = []; $res = Database::simpleQuery('SELECT subnetid, start, end, isdirect FROM reboot_subnet'); foreach ($res as $row) { $row += [ 'djumphosts' => [], 'ijumphosts' => [], ]; $subnets[$row['subnetid']] = $row; } // Get all jump hosts self::addJumphostsToSubnets($subnets); // Determine method for all clients $taskClients = []; // array of arrays with keys [ip, mac, methods] $taskSsh = []; // SSH configs for task, array of arrays with keys [username, sshkey, ip, port, command] $overrideBroadcast = Property::get(self::KEY_BROADCAST_ADDRESS); if (empty($overrideBroadcast)) { $overrideBroadcast = false; } foreach ($clientList as $dbClient) { $ip = sprintf('%u', ip2long($dbClient['clientip'])); // 32Bit snprintf unset($subnet); $subnet = false; foreach ($subnets as &$sn) { if ($sn['start'] <= $ip && $sn['end'] >= $ip) { $subnet =& $sn; break; } } if ($subnet === false) { $unknown[] = $dbClient; continue; } $taskClient = [ 'ip' => long2ip($subnet['end']), 'mac' => $dbClient['macaddr'], 'methods' => [], ]; // If we have an override broadcast address, unconditionally add this as the // first method if ($overrideBroadcast !== false) { $taskClient['ip'] = $overrideBroadcast; $taskClient['methods'][] = 'DIRECT'; } self::findMachinesForSubnet($subnet); // Highest priority - clients in same subnet, no directed broadcast // required, should be most reliable self::addSshMethodUsingClient($subnet['dclients'], $taskClient['methods'], $taskSsh); // Jumphost - usually in same subnet self::addSshMethodUsingJumphost($subnet['djumphosts'], true, $taskClient['methods'], $taskSsh); // Jumphosts in other subnets, determined to be able to reach destination subnet self::addSshMethodUsingJumphost($subnet['ijumphosts'], true, $taskClient['methods'], $taskSsh); // If directly reachable from server, prefer this now over the questionable approaches below, // but only if we didn't already add this above because of override if ($overrideBroadcast === false && $subnet['isdirect']) { $taskClient['methods'][] = 'DIRECT'; } // Use clients in other subnets, known to be able to reach the destination net self::addSshMethodUsingClient($subnet['iclients'], $taskClient['methods'], $taskSsh); // Add warning if nothing works if (empty($taskClient['methods'])) { $unreachable[] = $dbClient; } else { // TODO: Remember WOL was attempted } // "Questionable approaches": // Last fallback is jumphosts that were not reachable when last checked, this is really a last resort self::addSshMethodUsingJumphost($subnet['djumphosts'], false, $taskClient['methods'], $taskSsh); self::addSshMethodUsingJumphost($subnet['ijumphosts'], false, $taskClient['methods'], $taskSsh); if (!empty($taskClient['methods'])) { $taskClients[] = $taskClient; $sent[] = $dbClient; } } unset($subnet); if (!empty($unknown)) { $ips = ArrayUtil::flattenByKey($unknown, 'clientip'); $errors .= "**** WARNING ****\nThe following clients do not belong to a known subnet (bug?)\n" . implode("\n", $ips) . "\n"; foreach ($unknown as $val) { $failed[$val['clientip']] = $val; } } if (!empty($unreachable)) { $ips = ArrayUtil::flattenByKey($unreachable, 'clientip'); $errors .= "**** WARNING ****\nThe following clients are not reachable with any method\n" . implode("\n", $ips) . "\n"; foreach ($unreachable as $val) { $failed[$val['clientip']] = $val; } } $task = Taskmanager::submit('WakeOnLan', [ 'clients' => $taskClients, 'ssh' => $taskSsh, ]); if (isset($task['id'])) { $id = $task['id']; self::addTask($id, self::TASK_WOL, $clientList, ['log' => $errors]); foreach ($sent as $dbClient) { EventLog::applyFilterRules('#action-wol', $dbClient); } return $id; } return null; } private static function findMachinesForSubnet(&$subnet) { if (isset($subnet['dclients'])) return; $cutoff = time() - 320; // Get clients from same subnet first $subnet['dclients'] = Database::queryColumnArray("SELECT clientip FROM machine WHERE state IN ('IDLE', 'OCCUPIED') AND INET_ATON(clientip) BETWEEN :start AND :end AND lastseen > :cutoff LIMIT 3", ['start' => $subnet['start'], 'end' => $subnet['end'], 'cutoff' => $cutoff]); // If none, get clients from other subnets known to be able to reach this one $subnet['iclients'] = Database::queryColumnArray("SELECT m.clientip FROM reboot_subnet_x_subnet sxs INNER JOIN reboot_subnet s ON (s.subnetid = sxs.srcid AND sxs.dstid = :subnetid AND sxs.reachable = 1) INNER JOIN machine m ON (INET_ATON(m.clientip) BETWEEN s.start AND s.end AND state IN ('IDLE', 'OCCUPIED') AND m.lastseen > :cutoff) LIMIT 20", ['subnetid' => $subnet['subnetid'], 'cutoff' => $cutoff]); shuffle($subnet['iclients']); $subnet['iclients'] = array_slice($subnet['iclients'], 0, 3); } public static function prepareExec() { User::assertPermission('.rebootcontrol.action.exec'); $uuids = array_values(Request::post('uuid', Request::REQUIRED, 'array')); $machines = RebootUtils::getFilteredMachineList($uuids, '.rebootcontrol.action.exec'); if ($machines === false) return; $id = mt_rand(); Session::set('exec-' . $id, $machines, 60); Util::redirect('?do=rebootcontrol&show=exec&what=prepare&id=' . $id); } /** * Append a "wake via client" WOL method for the given client. Append at least one, but stop * if there are at least two methods already. * * @param array $sshClients [in] list of online clients to use for waking * @param array $c [out] The client's methods array * @param array $taskSsh [out] add according task struct to this array, if not already exists * @return void */ private static function addSshMethodUsingClient(array $sshClients, array &$methods, array &$taskSsh) { foreach ($sshClients as $host) { if (!isset($taskSsh[$host])) { $taskSsh[$host] = [ 'username' => 'root', 'sshkey' => SSHKey::getPrivateKey(), 'ip' => $host, 'port' => 9922, 'command' => self::buildClientWakeCommand('%MACS%', '%IP%'), ]; } $methods[] = $host; if (count($methods) >= 2) break; } } private static function addSshMethodUsingJumphost(array $jumpHosts, bool $reachable, array &$methods, array &$taskSsh) { // If it's the fallback to apparently unreachable jump-hosts, ignore if we already have two methods if (!$reachable && count($methods) >= 2) return; // host, port, username, sshkey, script, jh.reachable foreach ($jumpHosts as $jh) { if ($reachable !== (bool)$jh['reachable']) continue; $key = substr(md5($jh['host'] . ':' . $jh['port'] . ':' . $jh['username']), 0, 10); if (!isset($taskSsh[$key])) { $taskSsh[$key] = [ 'username' => $jh['username'], 'sshkey' => $jh['sshkey'], 'ip' => $jh['host'], 'port' => $jh['port'], 'command' => $jh['script'], ]; } $methods[] = $key; if (count($methods) >= 2) break; } } /** * Load all jumphosts from DB, sort into passed $subnets. Also split up * by directly assigned subnets, and indirectly dtermined, reachable subnets. * @param array $subnets [in] * @return void */ private static function addJumphostsToSubnets(array &$subnets) { $res = Database::simpleQuery('SELECT host, port, username, sshkey, script, jh.reachable, Group_Concat(jxs.subnetid) AS dsubnets, Group_Concat(sxs.dstid) AS isubnets FROM reboot_jumphost jh LEFT JOIN reboot_jumphost_x_subnet jxs ON (jh.hostid = jxs.hostid) LEFT JOIN reboot_subnet s ON (INET_ATON(jh.host) BETWEEN s.start AND s.end) LEFT JOIN reboot_subnet_x_subnet sxs ON (sxs.srcid = s.subnetid AND sxs.reachable <> 0) GROUP BY jh.hostid'); foreach ($res as $row) { $dnets = empty($row['dsubnets']) ? [] : explode(',', $row['dsubnets']); $inets = empty($row['isubnets']) ? [] : explode(',', $row['isubnets']); $inets = array_diff($inets, $dnets); // There might be duplicates if both joins match foreach ($dnets as $net) { if (empty($net) || !isset($subnets[$net])) continue; $subnets[$net]['djumphosts'][] =& $row; } foreach ($inets as $net) { if (empty($net) || !isset($subnets[$net])) continue; $subnets[$net]['ijumphosts'][] =& $row; } unset($row); } } }