path: root/modules-available/rebootcontrol/inc/rebootcontrol.inc.php

<?php

class RebootControl
{

	const KEY_TASKLIST = 'rebootcontrol.tasklist';

	const KEY_AUTOSCAN_DISABLED = 'rebootcontrol.disable.scan';

	const KEY_SCAN_CLIENT_TO_CLIENT = 'rebootcontrol.scan.c2c';

	const KEY_UDP_PORT = 'rebootcontrol.port';

	const KEY_BROADCAST_ADDRESS = 'rebootcontrol.broadcast-addr';

	const REBOOT = 'REBOOT';
	const KEXEC_REBOOT = 'KEXEC_REBOOT';
	const SHUTDOWN = 'SHUTDOWN';
	const TASK_REBOOTCTL = 'TASK_REBOOTCTL';
	const TASK_WOL = 'WAKE_ON_LAN';
	const TASK_EXEC = 'REMOTE_EXEC';

	/**
	 * @param string[] $uuids List of machineuuids to reboot
	 * @param bool $kexec whether to trigger kexec-reboot instead of full BIOS cycle
	 * @return false|array task struct for the reboot job
	 */
	public static function reboot(array $uuids, bool $kexec = false)
	{
		$list = RebootUtils::getMachinesByUuid($uuids);
		if (empty($list))
			return false;
		return self::execute($list, $kexec ? RebootControl::KEXEC_REBOOT : RebootControl::REBOOT, 0);
	}

	/**
	 * @param array $list list of clients containing each keys 'machineuuid', 'clientip' and 'locationid'
	 * @param string $mode reboot mode: RebootControl::REBOOT ::KEXEC_REBOOT or ::SHUTDOWN
	 * @param int $minutes delay in minutes for action
	 * @return array|false the task, or false if it could not be started
	 */
	public static function execute(array $list, string $mode, int $minutes)
	{
		$task = Taskmanager::submit("RemoteReboot", array(
			"clients" => $list,
			"mode" => $mode,
			"minutes" => $minutes,
			"sshkey" => SSHKey::getPrivateKey(),
			"port" => 9922, // Hard-coded, must match mgmt-sshd module
		));
		if (!Taskmanager::isFailed($task)) {
			self::addTask($task['id'], self::TASK_REBOOTCTL, $list, ['action' => $mode]);
			foreach ($list as $client) {
				$client['mode'] = $mode;
				$client['minutes'] = $minutes;
				EventLog::applyFilterRules('#action-power', $client);
			}
		}
		return $task;
	}

	/**
	 * Add wake task metadata to database, so we can display job details on the summary page.
	 */
	private static function addTask(string $taskId, string $type, array $clients, array $other = null): void
	{
		$lids = ArrayUtil::flattenByKey($clients, 'locationid');
		$lids = array_unique($lids);
		$newClients = [];
		foreach ($clients as $c) {
			$d = ['clientip' => $c['clientip']];
			if (isset($c['machineuuid'])) {
				$d['machineuuid'] = $c['machineuuid'];
			}
			$newClients[] = $d;
		}
		$data = [
			'id' => $taskId,
			'type' => $type,
			'locations' => $lids,
			'clients' => $newClients,
			'tasks' => [$taskId], // This did hold multiple tasks in the past; keep it in case we need this again
			'timestamp' => time(),
		];
		if (is_array($other)) {
			$data += $other;
		}
		Property::addToList(RebootControl::KEY_TASKLIST, json_encode($data), 20);
	}

	/**
	 * @param int[]|null $locations filter by these locations
	 * @param ?string $id only with this TaskID
	 * @return array|false list of active tasks for reboots/shutdowns.
	 */
	public static function getActiveTasks(array $locations = null, string $id = null)
	{
		if (is_array($locations) && in_array(0, $locations)) {
			$locations = null;
		}
		$list = Property::getList(RebootControl::KEY_TASKLIST);
		$return = [];
		foreach ($list as $subkey => $entry) {
			$p = json_decode($entry, true);
			if (!is_array($p) || !isset($p['id'])) {
				Property::removeFromListByKey(RebootControl::KEY_TASKLIST, $subkey);
				continue;
			}
			if (is_array($locations) && is_array($p['locations']) && array_diff($p['locations'], $locations) !== [])
				continue; // Not allowed
			if ($id !== null) {
				if ($p['id'] === $id)
					return $p;
				continue;
			}
			$valid = empty($p['tasks']);
			if (!$valid) {
				// Validate at least one task is still valid
				foreach ($p['tasks'] as $task) {
					$task = Taskmanager::status($task);
					if (Taskmanager::isTask($task)) {
						$p['status'] = $task['statusCode'];
						$valid = true;
						break;
					}
				}
			}
			if (!$valid) {
				Property::removeFromListByKey(RebootControl::KEY_TASKLIST, $subkey);
				continue;
			}
			$return[] = $p;
		}
		if ($id !== null)
			return false;
		return $return;
	}

	/**
	 * Execute given command or script on a list of hosts. The list of hosts is an array of structs containing
	 * each a known machine-uuid and/or hostname, and optionally a port to use, which would otherwise default to 9922,
	 * and optionally a username to use, which would default to root.
	 * The command should be compatible with the remote user's default shell (most likely bash).
	 *
	 * @param array $clients [ { clientip: <host>, machineuuid: <uuid>, port: <port>, username: <username> }, ... ]
	 * @param string $command Command or script to execute on client
	 * @param int $timeout in seconds
	 * @param string|false $privkey SSH private key to use to connect
	 * @return array|false task struct, false on error
	 */
	public static function runScript(array $clients, string $command, int $timeout = 5, $privkey = false)
	{
		$task = self::runScriptInternal($clients, $command, $timeout, $privkey);
		if (!Taskmanager::isFailed($task)) {
			self::addTask($task['id'], self::TASK_EXEC, $clients);
		}
		return $task;
	}

	private static function runScriptInternal(array &$clients, string $command, int $timeout = 5, $privkey = false)
	{
		$valid = [];
		$invalid = [];
		foreach ($clients as $client) {
			if (is_string($client)) {
				$invalid[strtoupper($client)] = []; // Assume machineuuid
			} elseif (!isset($client['clientip']) && !isset($client['machineuuid'])) {
				error_log('RebootControl::runScript called with list entry that has neither IP nor UUID');
			} elseif (!isset($client['clientip'])) {
				$invalid[$client['machineuuid']] = $client;
			} else {
				$valid[] = $client;
			}
		}
		if (!empty($invalid)) {
			$res = Database::simpleQuery('SELECT machineuuid, clientip, locationid FROM machine WHERE machineuuid IN (:uuids)',
				['uuids' => array_keys($invalid)]);
			foreach ($res as $row) {
				if (isset($invalid[$row['machineuuid']])) {
					$valid[] = $row + $invalid[$row['machineuuid']];
				} else {
					$valid[] = $row;
				}
			}
		}
		$clients = $valid;
		if (empty($clients)) {
			error_log('RebootControl::runScript called without any clients');
			return false;
		}
		if ($privkey === false) {
			$privkey = SSHKey::getPrivateKey();
		}
		return Taskmanager::submit('RemoteExec', [
			'clients' => $clients,
			'command' => $command,
			'timeoutSeconds' => $timeout,
			'sshkey' => $privkey,
			'port' => 9922, // Fallback if no port given in client struct
		]);
	}

	public static function connectionCheckCallback($task, $hostId)
	{
		$reachable = 0;
		if (isset($task['data']['result'])) {
			foreach ($task['data']['result'] as $res) {
				if ($res['exitCode'] == 0) {
					$reachable = 1;
				}
			}
		}
		Database::exec('UPDATE reboot_jumphost SET reachable = :reachable WHERE hostid = :id',
			['id' => $hostId, 'reachable' => $reachable]);
	}

	/**
	 * Wake clients given by MAC address(es) via jawol util.
	 * Multiple MAC addresses can be passed as a space separated list.
	 */
	private static function buildClientWakeCommand(string $macs, string $bcast = null, string $passwd = null): string
	{
		$command = 'jawol';
		if (!empty($bcast)) {
			$command .= " -d '$bcast'";
		} else {
			$command .= ' -i br0';
		}
		if (!empty($passwd)) {
			$command .= " -p '$passwd'";
		}
		$command .= " $macs";
		return $command;
	}

	/**
	 * @param array $sourceMachines list of source machines. array of [clientip, machineuuid] entries
	 * @param string $macaddr destination mac address(es)
	 * @param string $bcast directed broadcast address to send to
	 * @param string $passwd optional WOL password, mac address or ipv4 notation
	 * @return array|false task struct, false on error
	 */
	public static function wakeViaClient(array $sourceMachines, string $macaddr, string $bcast = null, string $passwd = null)
	{
		$command = self::buildClientWakeCommand($macaddr, $bcast, $passwd);
		// Yes there is one zero "missing" from the usleep -- that's the whole point: we prefer 100ms sleeps
		return self::runScriptInternal($sourceMachines,
			"for i in 1 1 0; do $command; usleep \${i}00000 2> /dev/null || sleep \$i; done");
	}

	/**
	 * @param string|string[] $macaddr destination mac address(es)
	 * @param ?string $bcast directed broadcast address to send to
	 * @param ?string $passwd optional WOL password; mac address or ipv4 notation
	 * @return array|false task struct, false on error
	 */
	public static function wakeDirectly($macaddr, string $bcast = null, string $passwd = null)
	{
		if (!is_array($macaddr)) {
			$macaddr = [$macaddr];
		}
		$port = (int)Property::get(RebootControl::KEY_UDP_PORT);
		if ($port < 1 || $port > 65535) {
			$port = 9;
		}
		$arg = [];
		foreach ($macaddr as $mac) {
			$arg[] = [
				'ip' => $bcast,
				'mac' => $mac,
				'methods' => ['DIRECT'],
				'password' => $passwd,
			];
		}
		return Taskmanager::submit('WakeOnLan', ['clients' => $arg]);
	}

	/**
	 * Explicitly wake given clients via jumphost
	 * @param array $jumphost the according row from the database, representing the desired jumphost
	 * @param string $bcast (directed) broadcast address for WOL packet, %IP% in command template
	 * @param array $clients list of clients, must contain at least key 'macaddr' for every client
	 * @return array|false task struct on successful submission to TM, false on error
	 */
	public static function wakeViaJumpHost(array $jumphost, string $bcast, array $clients)
	{
		$hostid = $jumphost['hostid'];
		$macs = ArrayUtil::flattenByKey($clients, 'macaddr');
		if (empty($macs)) {
			error_log('Called wakeViaJumpHost without clients');
			return false;
		}
		$macs = "'" . implode("' '", $macs) . "'";
		$macs = str_replace('-', ':', $macs);
		$script = str_replace(['%IP%', '%MACS%'], [$bcast, $macs], $jumphost['script']);
		$arg = [[
			'clientip' => $jumphost['host'],
			'port' => $jumphost['port'],
			'username' => $jumphost['username'],
		]];
		$task = RebootControl::runScriptInternal($arg, $script, 6, $jumphost['sshkey']);
		if ($task !== false && isset($task['id'])) {
			TaskmanagerCallback::addCallback($task, 'rbcConnCheck', $hostid);
		}
		return $task;
	}

	/**
	 * @param array $clientList list of clients containing each keys 'macaddr' and 'clientip', optionally 'locationid'
	 * @param ?array $failed list of failed clients from $clientList
	 * @return ?string taskid of this job
	 */
	public static function wakeMachines(array $clientList, array &$failed = null): ?string
	{
		$errors = '';
		$sent = $unknown = $unreachable = $failed = [];
		// For event filtering by rule
		// Need all subnets...
		/* subnetid => [
		 * subnetid => 1234,
		 * start => 1234, (ip2long)
		 * end => 5678, (ip2long)
		 * jumphosts => [id1, id2, ...],
		 */
		$subnets = [];
		$res = Database::simpleQuery('SELECT subnetid, start, end, isdirect FROM reboot_subnet');
		foreach ($res as $row) {
			$row += [
				'djumphosts' => [],
				'ijumphosts' => [],
			];
			$subnets[$row['subnetid']] = $row;
		}
		// Get all jump hosts
		self::addJumphostsToSubnets($subnets);
		// Determine method for all clients
		$taskClients = []; // array of arrays with keys [ip, mac, methods]
		$taskSsh = []; // SSH configs for task, array of arrays with keys [username, sshkey, ip, port, command]
		$overrideBroadcast = Property::get(self::KEY_BROADCAST_ADDRESS);
		if (empty($overrideBroadcast)) {
			$overrideBroadcast = false;
		}
		foreach ($clientList as $dbClient) {
			$ip = sprintf('%u', ip2long($dbClient['clientip'])); // 32Bit snprintf
			unset($subnet);
			$subnet = false;
			foreach ($subnets as &$sn) {
				if ($sn['start'] <= $ip && $sn['end'] >= $ip) {
					$subnet =& $sn;
					break;
				}
			}
			if ($subnet === false) {
				$unknown[] = $dbClient;
				continue;
			}
			$taskClient = [
				'ip' => long2ip($subnet['end']),
				'mac' => $dbClient['macaddr'],
				'methods' => [],
			];
			// If we have an override broadcast address, unconditionally add this as the
			// first method
			if ($overrideBroadcast !== false) {
				$taskClient['ip'] = $overrideBroadcast;
				$taskClient['methods'][] = 'DIRECT';
			}
			self::findMachinesForSubnet($subnet);
			// Highest priority - clients in same subnet, no directed broadcast
			// required, should be most reliable
			self::addSshMethodUsingClient($subnet['dclients'], $taskClient['methods'], $taskSsh);
			// Jumphost - usually in same subnet
			self::addSshMethodUsingJumphost($subnet['djumphosts'], true, $taskClient['methods'], $taskSsh);
			// Jumphosts in other subnets, determined to be able to reach destination subnet
			self::addSshMethodUsingJumphost($subnet['ijumphosts'], true, $taskClient['methods'], $taskSsh);
			// If directly reachable from server, prefer this now over the questionable approaches below,
			// but only if we didn't already add this above because of override
			if ($overrideBroadcast === false && $subnet['isdirect']) {
				$taskClient['methods'][] = 'DIRECT';
			}
			// Use clients in other subnets, known to be able to reach the destination net
			self::addSshMethodUsingClient($subnet['iclients'], $taskClient['methods'], $taskSsh);
			// Add warning if nothing works
			if (empty($taskClient['methods'])) {
				$unreachable[] = $dbClient;
			} else {
				// TODO: Remember WOL was attempted
			}
			// "Questionable approaches":
			// Last fallback is jumphosts that were not reachable when last checked, this is really a last resort
			self::addSshMethodUsingJumphost($subnet['djumphosts'], false, $taskClient['methods'], $taskSsh);
			self::addSshMethodUsingJumphost($subnet['ijumphosts'], false, $taskClient['methods'], $taskSsh);

			if (!empty($taskClient['methods'])) {
				$taskClients[] = $taskClient;
				$sent[] = $dbClient;
			}
		}
		unset($subnet);

		if (!empty($unknown)) {
			$ips = ArrayUtil::flattenByKey($unknown, 'clientip');
			$errors .= "**** WARNING ****\nThe following clients do not belong to a known subnet (bug?)\n" . implode("\n", $ips) . "\n";
			foreach ($unknown as $val) {
				$failed[$val['clientip']] = $val;
			}
		}
		if (!empty($unreachable)) {
			$ips = ArrayUtil::flattenByKey($unreachable, 'clientip');
			$errors .= "**** WARNING ****\nThe following clients are not reachable with any method\n" . implode("\n", $ips) . "\n";
			foreach ($unreachable as $val) {
				$failed[$val['clientip']] = $val;
			}
		}
		$task = Taskmanager::submit('WakeOnLan', [
			'clients' => $taskClients,
			'ssh' => $taskSsh,
		]);
		if (isset($task['id'])) {
			$id = $task['id'];
			self::addTask($id, self::TASK_WOL, $clientList, ['log' => $errors]);
			foreach ($sent as $dbClient) {
				EventLog::applyFilterRules('#action-wol', $dbClient);
			}
			return $id;
		}
		return null;
	}

	private static function findMachinesForSubnet(&$subnet)
	{
		if (isset($subnet['dclients']))
			return;
		$cutoff = time() - 320;
		// Get clients from same subnet first
		$subnet['dclients'] = Database::queryColumnArray("SELECT clientip FROM machine
			WHERE state IN ('IDLE', 'OCCUPIED') AND INET_ATON(clientip) BETWEEN :start AND :end AND lastseen > :cutoff
			LIMIT 3",
			['start' => $subnet['start'], 'end' => $subnet['end'], 'cutoff' => $cutoff]);
		// If none, get clients from other subnets known to be able to reach this one
		$subnet['iclients'] = Database::queryColumnArray("SELECT m.clientip FROM reboot_subnet_x_subnet sxs
    		INNER JOIN reboot_subnet s ON (s.subnetid = sxs.srcid AND sxs.dstid = :subnetid AND sxs.reachable = 1)
			INNER JOIN machine m ON (INET_ATON(m.clientip) BETWEEN s.start AND s.end AND state IN ('IDLE', 'OCCUPIED') AND m.lastseen > :cutoff)
			LIMIT 20", ['subnetid' => $subnet['subnetid'], 'cutoff' => $cutoff]);
		shuffle($subnet['iclients']);
		$subnet['iclients'] = array_slice($subnet['iclients'], 0, 3);
	}

	public static function prepareExec()
	{
		User::assertPermission('.rebootcontrol.action.exec');
		$uuids = array_values(Request::post('uuid', Request::REQUIRED, 'array'));
		$machines = RebootUtils::getFilteredMachineList($uuids, '.rebootcontrol.action.exec');
		if ($machines === false)
			return;
		$id = mt_rand();
		Session::set('exec-' . $id, $machines, 60);
		Util::redirect('?do=rebootcontrol&show=exec&what=prepare&id=' . $id);
	}

	/**
	 * Append a "wake via client" WOL method for the given client. Append at least one, but stop
	 * if there are at least two methods already.
	 *
	 * @param array $sshClients [in] list of online clients to use for waking
	 * @param array $c [out] The client's methods array
	 * @param array $taskSsh [out] add according task struct to this array, if not already exists
	 * @return void
	 */
	private static function addSshMethodUsingClient(array $sshClients, array &$methods, array &$taskSsh)
	{
		foreach ($sshClients as $host) {
			if (!isset($taskSsh[$host])) {
				$taskSsh[$host] = [
					'username' => 'root',
					'sshkey' => SSHKey::getPrivateKey(),
					'ip' => $host,
					'port' => 9922,
					'command' => self::buildClientWakeCommand('%MACS%', '%IP%'),
				];
			}
			$methods[] = $host;
			if (count($methods) >= 2)
				break;
		}
	}

	private static function addSshMethodUsingJumphost(array $jumpHosts, bool $reachable, array &$methods, array &$taskSsh)
	{
		// If it's the fallback to apparently unreachable jump-hosts, ignore if we already have two methods
		if (!$reachable && count($methods) >= 2)
			return;
		// host, port, username, sshkey, script, jh.reachable
		foreach ($jumpHosts as $jh) {
			if ($reachable !== (bool)$jh['reachable'])
				continue;
			$key = substr(md5($jh['host'] . ':' . $jh['port'] . ':' . $jh['username']), 0, 10);
			if (!isset($taskSsh[$key])) {
				$taskSsh[$key] = [
					'username' => $jh['username'],
					'sshkey' => $jh['sshkey'],
					'ip' => $jh['host'],
					'port' => $jh['port'],
					'command' => $jh['script'],
				];
			}
			$methods[] = $key;
			if (count($methods) >= 2)
				break;
		}
	}

	/**
	 * Load all jumphosts from DB, sort into passed $subnets. Also split up
	 * by directly assigned subnets, and indirectly dtermined, reachable subnets.
	 * @param array $subnets [in]
	 * @return void
	 */
	private static function addJumphostsToSubnets(array &$subnets)
	{
		$res = Database::simpleQuery('SELECT host, port, username, sshkey, script, jh.reachable,
       	Group_Concat(jxs.subnetid) AS dsubnets, Group_Concat(sxs.dstid) AS isubnets
			FROM reboot_jumphost jh
			LEFT JOIN reboot_jumphost_x_subnet jxs ON (jh.hostid = jxs.hostid)
			LEFT JOIN reboot_subnet s ON (INET_ATON(jh.host) BETWEEN s.start AND s.end)
			LEFT JOIN reboot_subnet_x_subnet sxs ON (sxs.srcid = s.subnetid AND sxs.reachable <> 0)
			GROUP BY jh.hostid');
		foreach ($res as $row) {
			$dnets = empty($row['dsubnets']) ? [] : explode(',', $row['dsubnets']);
			$inets = empty($row['isubnets']) ? [] : explode(',', $row['isubnets']);
			$inets = array_diff($inets, $dnets); // There might be duplicates if both joins match
			foreach ($dnets as $net) {
				if (empty($net) || !isset($subnets[$net]))
					continue;
				$subnets[$net]['djumphosts'][] =& $row;
			}
			foreach ($inets as $net) {
				if (empty($net) || !isset($subnets[$net]))
					continue;
				$subnets[$net]['ijumphosts'][] =& $row;
			}
			unset($row);
		}
	}

}