From 0156024a414ea1503e539cb2a30d05a422d4cd16 Mon Sep 17 00:00:00 2001 From: Simon Rettberg Date: Tue, 21 Sep 2021 16:52:06 +0200 Subject: Passthrough WIP --- .../minilinux/inc/linuxbootentryhook.inc.php | 20 +++-- modules-available/passthrough/config.json | 7 ++ .../passthrough/inc/passthrough.inc.php | 42 ++++++++++ modules-available/passthrough/install.inc.php | 9 ++ modules-available/passthrough/page.inc.php | 97 ++++++++++++++++++++++ .../passthrough/permissions/permissions.json | 3 + .../passthrough/templates/hardware-list.html | 96 +++++++++++++++++++++ modules-available/statistics/api.inc.php | 4 +- .../statistics/inc/hardwareinfo.inc.php | 54 ++++++++++++ .../statistics/inc/hardwareparser.inc.php | 20 ++--- .../statistics/inc/hardwarequery.inc.php | 40 ++++++--- modules-available/statistics/inc/pciid.inc.php | 63 ++++++++++++++ modules-available/statistics/page.inc.php | 50 ++--------- 13 files changed, 432 insertions(+), 73 deletions(-) create mode 100644 modules-available/passthrough/config.json create mode 100644 modules-available/passthrough/inc/passthrough.inc.php create mode 100644 modules-available/passthrough/install.inc.php create mode 100644 modules-available/passthrough/page.inc.php create mode 100644 modules-available/passthrough/permissions/permissions.json create mode 100644 modules-available/passthrough/templates/hardware-list.html create mode 100644 modules-available/statistics/inc/pciid.inc.php diff --git a/modules-available/minilinux/inc/linuxbootentryhook.inc.php b/modules-available/minilinux/inc/linuxbootentryhook.inc.php index e57336f0..9e10bd6e 100644 --- a/modules-available/minilinux/inc/linuxbootentryhook.inc.php +++ b/modules-available/minilinux/inc/linuxbootentryhook.inc.php @@ -15,7 +15,7 @@ class LinuxBootEntryHook extends BootEntryHook return Dictionary::translateFileModule('minilinux', 'module', 'module_name', true); } - public function extraFields() + public function extraFields(): array { /* For translate module: * Dictionary::translate('ipxe-kcl-extra'); @@ -32,7 +32,7 @@ class LinuxBootEntryHook extends BootEntryHook /** * @return HookEntryGroup[] */ - protected function groupsInternal() + protected function groupsInternal(): array { /* * Dictionary::translate('default_boot_entry'); @@ -76,7 +76,7 @@ class LinuxBootEntryHook extends BootEntryHook * @param $localData * @return BootEntry the actual boot entry instance for given entry, false if invalid id */ - public function getBootEntryInternal($localData) + public function getBootEntryInternal($localData): BootEntry { $id = $localData['id']; if ($id === 'default') { // Special case @@ -125,7 +125,7 @@ class LinuxBootEntryHook extends BootEntryHook return BootEntry::newStandardBootEntry($bios, $efi, $arch); } - private function generateExecData($effectiveId, $remoteData, $localData) + private function generateExecData($effectiveId, $remoteData, $localData): ExecData { $exec = new ExecData(); // Defaults @@ -146,9 +146,7 @@ class LinuxBootEntryHook extends BootEntryHook if (!empty($localData['debug'])) { // Debug boot enabled $exec->commandLine = IPxe::modifyCommandLine($exec->commandLine, - isset($remoteData['debugCommandLineModifier']) - ? $remoteData['debugCommandLineModifier'] - : '-vga -quiet -splash -loglevel loglevel=7' + $remoteData['debugCommandLineModifier'] ?? '-vga -quiet -splash -loglevel loglevel=7' ); } // disable all CPU sidechannel attack mitigations etc. @@ -156,6 +154,14 @@ class LinuxBootEntryHook extends BootEntryHook $exec->commandLine = IPxe::modifyCommandLine($exec->commandLine, 'noibrs noibpb nopti nospectre_v2 nospectre_v1 l1tf=off nospec_store_bypass_disable no_stf_barrier mds=off mitigations=off'); } + // GVT, PCI Pass-thru etc. + if (Module::isAvailable('statistics')) { + $hwextra = HardwareInfo::getKclModifications(); + if (!empty($hwextra)) { + $exec->commandLine = IPxe::modifyCommandLine($exec->commandLine, $hwextra); + } + } + // User-supplied modifications if (!empty($localData['kcl-extra'])) { $exec->commandLine = IPxe::modifyCommandLine($exec->commandLine, $localData['kcl-extra']); } diff --git a/modules-available/passthrough/config.json b/modules-available/passthrough/config.json new file mode 100644 index 00000000..3f65995f --- /dev/null +++ b/modules-available/passthrough/config.json @@ -0,0 +1,7 @@ +{ + "category": "main.settings-client", + "dependencies": [ + "statistics", + "locations" + ] +} \ No newline at end of file diff --git a/modules-available/passthrough/inc/passthrough.inc.php b/modules-available/passthrough/inc/passthrough.inc.php new file mode 100644 index 00000000..51fe7214 --- /dev/null +++ b/modules-available/passthrough/inc/passthrough.inc.php @@ -0,0 +1,42 @@ + $title) { + if ($row['class'] !== '0300' && ($id === 'GPU' || $id === 'GVT')) + continue; + $item = ['ptid' => $id, 'ptname' => $id . ' (' . $title . ')']; + if ($row['@PASSTHROUGH'] === $id) { + $item['selected'] = 'selected'; + } + $out[] = $item; + } + return $out; + } + + private static function ensurePrepopulated(&$list) + { + $want = [ + 'GPU' => '[Special] GPU passthrough default group', + 'GVT' => '[Special] Intel GVT-g default group', + ]; + foreach ($want as $id => $title) { + if (!isset($list[$id])) { + Database::exec("INSERT INTO passthrough_group (groupid, title) VALUES (:id, :title) + ON DUPLICATE KEY UPDATE title = VALUES(title)", + ['id' => $id, 'title' => $title]); + $list[$id] = $title; + } + } + } + +} \ No newline at end of file diff --git a/modules-available/passthrough/install.inc.php b/modules-available/passthrough/install.inc.php new file mode 100644 index 00000000..e08be38b --- /dev/null +++ b/modules-available/passthrough/install.inc.php @@ -0,0 +1,9 @@ +saveHwList(); + } + if (Request::isPost()) { + Util::redirect('?do=passthrough'); + } + } + + private function saveHwList() + { + $newgroups = Request::post('newgroup', [], 'array'); + foreach ($newgroups as $id => $title) { + $id = strtoupper(preg_replace('/[^a-z0-9_\-]/ig', '', $id)); + if (empty($id)) + continue; + Database::exec("INSERT IGNORE INTO passthrough_group (groupid, title) + VALUES (:group, :title)", + ['group' => $id, 'title' => $title]); + } + $groups = Request::post('ptgroup', Request::REQUIRED, 'array'); + $insert = []; + $delete = []; + foreach ($groups as $hwid => $group) { + if (empty($group)) { + $delete[] = $hwid; + } else { + $insert[] = [$hwid, '@PASSTHROUGH', $group]; + } + } + if (!empty($insert)) { + Database::exec("INSERT INTO statistic_hw_prop (hwid, prop, `value`) VALUES :list + ON DUPLICATE KEY UPDATE `value` = VALUES(`value`)", + ['list' => $insert]); + } + if (!empty($delete)) { + Database::exec("DELETE FROM statistic_hw_prop WHERE hwid IN (:list) AND prop = '@PASSTHROUGH'", ['list' => $delete]); + } + Message::addSuccess('list-updated'); + Util::redirect('?do=passthrough&show=hwlist'); + } + + /* + * + */ + + protected function doRender() + { + $show = Request::get('show'); + if ($show === 'hwlist') { + $this->showHardwareList(); + } else { + Util::redirect('?do=passthrough&show=hwlist'); + } + } + + private function showHardwareList() + { + $q = new HardwareQuery(HardwareInfo::PCI_DEVICE, null, false); + $q->addColumn(true, 'vendor'); + $q->addColumn(true, 'device'); + $q->addColumn(true, 'class'); + $q->addColumn(true, '@PASSTHROUGH'); + $rows = []; + foreach ($q->query('shw.hwid') as $row) { + $row['ptlist'] = Passthrough::getGroupDropdown($row); + $rows[] = $row; + } + // Sort Graphics Cards first, rest by class, vendor, device + usort($rows, function ($row1, $row2) { + $a = $row1['class']; + $b = $row2['class']; + if ($a === $b) + return hexdec($row1['vendor'].$row1['device']) - hexdec($row2['vendor'] . $row2['device']); + if ($a === '0300') + return -1; + if ($b === '0300') + return 1; + return hexdec($a) - hexdec($b); + }); + foreach ($rows as &$row) { + $row['vendor_name'] = PciId::getPciId(PciId::VENDOR, $row['vendor'] ?? ''); + $row['device_name'] = PciId::getPciId(PciId::DEVICE, $row['vendor'] . ':' . $row['device']); + } + Render::addTemplate('hardware-list', ['list' => $rows]); + } + +} \ No newline at end of file diff --git a/modules-available/passthrough/permissions/permissions.json b/modules-available/passthrough/permissions/permissions.json new file mode 100644 index 00000000..d0932f1e --- /dev/null +++ b/modules-available/passthrough/permissions/permissions.json @@ -0,0 +1,3 @@ +{ + "view": false +} \ No newline at end of file diff --git a/modules-available/passthrough/templates/hardware-list.html b/modules-available/passthrough/templates/hardware-list.html new file mode 100644 index 00000000..2450e457 --- /dev/null +++ b/modules-available/passthrough/templates/hardware-list.html @@ -0,0 +1,96 @@ +
+ + + + + + + + + + + + + {{#list}} + + + + + + + + {{/list}} + +
{{lang_class}}{{lang_deviceIdNumeric}}{{lang_deviceName}}{{lang_useCount}}{{lang_passthroughGroup}}
{{class}}{{vendor}}:{{device}} + + + + +
{{device_name}}
+
{{vendor_name}}
+ 		{{connected_count}} + +
+
+
+ + +
+
+ + + + \ No newline at end of file diff --git a/modules-available/statistics/api.inc.php b/modules-available/statistics/api.inc.php index 0a194f2e..e20ae696 100644 --- a/modules-available/statistics/api.inc.php +++ b/modules-available/statistics/api.inc.php @@ -1,6 +1,8 @@ addCompare(false, 'Memory Slot Occupied', '>=', true, 'Memory Slot Count'); $x->addWhere(true, 'vendor', '=', '8086'); @@ -79,7 +81,7 @@ if ($type[0] === '~') { $hostname = ''; } $data = Util::cleanUtf8(Request::post('json', '', 'string')); - $hasJson = !empty($data); + $hasJson = !empty($data) && $data[0] === '{'; if (!$hasJson) { $data = Util::cleanUtf8(Request::post('data', '', 'string')); } diff --git a/modules-available/statistics/inc/hardwareinfo.inc.php b/modules-available/statistics/inc/hardwareinfo.inc.php index 5ef94365..90b8975b 100644 --- a/modules-available/statistics/inc/hardwareinfo.inc.php +++ b/modules-available/statistics/inc/hardwareinfo.inc.php @@ -13,4 +13,58 @@ class HardwareInfo const HDD = 'HDD'; const CPU = 'CPU'; + /** + * Get a KCL modification string for the given machine, enabling GVT, PCI passthrough etc. + * You can provide a UUID and/or MAC, or nothing. If nothing is provided, + * the "uuid" and "mac" GET parameters will be used. If both are provided, + * the resulting machine that has the greatest "lastseen" value will be used. + * @param ?string $uuid UUID of machine + * @param ?string $mac MAC of machine + * @return string + */ + public static function getKclModifications($uuid = null, $mac = null): string + { + if ($uuid === null && $mac === null) { + $uuid = Request::get('uuid', '', 'string'); + $mac = Request::get('mac', '', 'string'); + $mac = str_replace(':', '-', $mac); + } + $res = Database::simpleQuery("SELECT machineuuid, lastseen FROM machine + WHERE machineuuid = :uuid OR macaddr = :mac", ['uuid' => $uuid, 'mac' => $mac]); + $best = null; + foreach ($res as $row) { + if ($best === null || $best['lastseen'] < $row['lastseen']) { + $best = $row; + } + } + if ($best === null) + return ''; + $hw = new HardwareQuery(self::PCI_DEVICE, $best['machineuuid'], true); + // TODO: Get list of enabled pass through groups for this client's location + $hw->addWhere(true, '@PASSTHROUGH', 'IN', ['GPU', 'GVT']); + $hw->addColumn(true, 'vendor'); + $hw->addColumn(true, 'device'); + $res = $hw->query(); + $passthrough = []; + $gvt = false; + foreach ($res as $row) { + if ($row['@PASSTHROUGH'] === 'GVT') { + $gvt = true; + } else { + $passthrough[] = $row['vendor'] . ':' . $row['device']; + } + } + $kcl = ''; + if ($gvt || !empty($passthrough)) { + $kcl = '-iommu -intel_iommu iommu=pt intel_iommu=on'; // TODO AMD + } + if (!empty($passthrough)) { + $kcl .= ' vfio-pci.ids=' . implode(',', $passthrough); + } + if ($gvt) { + $kcl .= ' i915.enable_gvt=1'; + } + return $kcl; + } + } diff --git a/modules-available/statistics/inc/hardwareparser.inc.php b/modules-available/statistics/inc/hardwareparser.inc.php index d356e226..87dcc4cf 100644 --- a/modules-available/statistics/inc/hardwareparser.inc.php +++ b/modules-available/statistics/inc/hardwareparser.inc.php @@ -308,29 +308,29 @@ class HardwareParser foreach ($out as $entry) { if (!isset($pci[$entry['class']])) { $class = 'c.' . $entry['class']; - $res = Page_Statistics::getPciId('CLASS', $class); - if ($res === false || $res['dateline'] < $NOW) { + $res = PciId::getPciId('CLASS', $class); + if ($res === false) { $pci[$entry['class']]['lookupClass'] = 'do-lookup'; $pci[$entry['class']]['class'] = $class; } else { - $pci[$entry['class']]['class'] = $res['value']; + $pci[$entry['class']]['class'] = $res; } } $new = array( 'ven' => $entry['ven'], 'dev' => $entry['ven'] . ':' . $entry['dev'], ); - $res = Page_Statistics::getPciId('VENDOR', $new['ven']); - if ($res === false || $res['dateline'] < $NOW) { + $res = PciId::getPciId('VENDOR', $new['ven']); + if ($res === false) { $new['lookupVen'] = 'do-lookup'; } else { - $new['ven'] = $res['value']; + $new['ven'] = $res; } - $res = Page_Statistics::getPciId('DEVICE', $new['dev']); - if ($res === false || $res['dateline'] < $NOW) { + $res = PciId::getPciId('DEVICE', $new['dev']); + if ($res === false) { $new['lookupDev'] = 'do-lookup'; } else { - $new['dev'] = $res['value'] . ' (' . $new['dev'] . ')'; + $new['dev'] = $res . ' (' . $new['dev'] . ')'; } $pci[$entry['class']]['entries'][] = $new; } @@ -717,7 +717,7 @@ class HardwareParser $hwid = self::writeGlobalHardwareData(HardwareInfo::PCI_DEVICE, self::propsFromArray($dev, 'vendor', 'device', 'rev', 'class')); $mappingId = self::writeLocalHardwareData($uuid, $hwid, $dev['slot'] ?? 'unknown', - self::propsFromArray($dev, 'slot', 'subsystem', 'subsystem_vendor')); + self::propsFromArray($dev, 'slot', 'subsystem', 'subsystem_vendor', 'iommu_group')); $pciHwIds[] = $mappingId; } self::markDisconnected($uuid, HardwareInfo::PCI_DEVICE, $pciHwIds); diff --git a/modules-available/statistics/inc/hardwarequery.inc.php b/modules-available/statistics/inc/hardwarequery.inc.php index 7ccde2f6..b0b7d6ee 100644 --- a/modules-available/statistics/inc/hardwarequery.inc.php +++ b/modules-available/statistics/inc/hardwarequery.inc.php @@ -16,13 +16,15 @@ class HardwareQuery public function __construct($type, $uuid = null, $connectedOnly = true) { if ($connectedOnly) { - $this->where[] = 'mxhw.disconnecttime = 0'; + $this->joins['mxhw_join'] = "INNER JOIN machine_x_hw mxhw ON (mxhw.hwid = shw.hwid AND mxhw.disconnecttime = 0)"; + } else { + $this->joins['mxhw_join'] = "INNER JOIN machine_x_hw mxhw ON (mxhw.hwid = shw.hwid)"; } if ($uuid !== null) { $this->where[] = 'mxhw.machineuuid = :uuid'; $this->args['uuid'] = $uuid; } - $this->joins[] = "INNER JOIN statistic_hw shw ON (mxhw.hwid = shw.hwid AND shw.hwtype = :hwtype)"; + $this->where[] = 'shw.hwtype = :hwtype'; $this->args['hwtype'] = $type; } @@ -44,7 +46,13 @@ class HardwareQuery } } - public function addWhere(bool $global, string $prop, string $op, string $value) + /** + * @param bool $global + * @param string $prop + * @param string $op + * @param string|string[] $value + */ + public function addWhere(bool $global, string $prop, string $op, $value) { if (isset($this->columns[$prop])) return; @@ -54,7 +62,7 @@ class HardwareQuery $vid = $this->id(); $valueCol = ($op === '<' || $op === '>' || $op === '<=' || $op === '>=') ? 'numeric' : 'value'; $this->joins[$prop] = "INNER JOIN $table $tid ON ($srcTable.$column = $tid.$column AND - $tid.prop = :$pid AND $tid.`$valueCol` $op :$vid)"; + $tid.prop = :$pid AND $tid.`$valueCol` $op (:$vid))"; $this->args[$pid] = $prop; $this->args[$vid] = $value; $this->columns[$prop] = "$tid.`value` AS `$prop`"; @@ -85,7 +93,7 @@ class HardwareQuery $this->fillTableVars($global, $srcTable, $table, $column); $tid = $this->id(); $pid = $this->id(); - $this->joins[$prop] = "INNER JOIN $table $tid ON ($srcTable.$column = $tid.$column AND $tid.prop = :$pid)"; + $this->joins[$prop] = "LEFT JOIN $table $tid ON ($srcTable.$column = $tid.$column AND $tid.prop = :$pid)"; $this->args[$pid] = $prop; $this->columns[$prop] = "$tid.`value` AS `$prop`"; } @@ -93,13 +101,25 @@ class HardwareQuery /** * @return false|PDOStatement */ - public function query() + public function query(string $groupBy = '') { - $this->columns[] = 'mxhw.machineuuid'; - $query = 'SELECT ' . implode(', ', $this->columns) - . ' FROM machine_x_hw mxhw ' + $columns = $this->columns; + $columns[] = 'mxhw.machineuuid'; + $columns[] = 'shw.hwid'; + if (empty($groupBy) || $groupBy === 'mxhw.machinehwid') { + $columns[] = 'mxhw.disconnecttime'; + } else { + $columns[] = 'Sum(If(mxhw.disconnecttime = 0, 1, 0)) AS connected_count'; + } + if (!empty($groupBy)) { + $columns[] = 'Count(*) AS group_count'; + $groupBy = " GROUP BY $groupBy"; + } + $query = 'SELECT ' . implode(', ', $columns) + . ' FROM statistic_hw shw ' . implode(' ', $this->joins) - . ' WHERE ' . implode(' AND ', $this->where); + . ' WHERE ' . implode(' AND ', $this->where) + . $groupBy; return Database::simpleQuery($query, $this->args); } diff --git a/modules-available/statistics/inc/pciid.inc.php b/modules-available/statistics/inc/pciid.inc.php new file mode 100644 index 00000000..6bf852e6 --- /dev/null +++ b/modules-available/statistics/inc/pciid.inc.php @@ -0,0 +1,63 @@ + $cat, 'id' => $id)); + if ($row !== false && $row['dateline'] >= time()) { + return $cache[$key] = $row['value']; + } + if (!$dnsQuery) + return false; + // Unknown, query + if ($cat === self::DEVICE && preg_match('/^([a-f0-9]{4}):([a-f0-9]{4})$/', $id, $out)) { + $host = $out[2] . '.' . $out[1]; + } elseif ($cat === self::VENDOR && preg_match('/^([a-f0-9]{4})$/', $id, $out)) { + $host = $out[1]; + } elseif ($cat === self::DEVCLASS && preg_match('/^c\.([a-f0-9]{2})([a-f0-9]{2})$/', $id, $out)) { + $host = $out[2] . '.' . $out[1] . '.c'; + } else { + error_log("getPciId called with unknown format: ($cat) ($id)"); + return false; + } + $res = dns_get_record($host . '.pci.id.ucw.cz', DNS_TXT); + if (!is_array($res)) + return false; + foreach ($res as $entry) { + if (isset($entry['txt']) && substr($entry['txt'], 0, 2) === 'i=') { + $string = substr($entry['txt'], 2); + Database::exec('INSERT INTO pciid (category, id, value, dateline) VALUES (:cat, :id, :value, :timeout)' + . ' ON DUPLICATE KEY UPDATE value = VALUES(value), dateline = VALUES(dateline)', + array( + 'cat' => $cat, + 'id' => $id, + 'value' => $string, + 'timeout' => time() + mt_rand(10, 30) * 86400, + ), true); + return $cache[$key] = $string; + } + } + return $cache[$key] = ($row['value'] ?? false); + } + +} \ No newline at end of file diff --git a/modules-available/statistics/page.inc.php b/modules-available/statistics/page.inc.php index 3e4aa9ce..d26649b6 100644 --- a/modules-available/statistics/page.inc.php +++ b/modules-available/statistics/page.inc.php @@ -264,60 +264,20 @@ class Page_Statistics extends Page $add = ''; if (preg_match('/^([a-f0-9]{4}):([a-f0-9]{4})$/', $param, $out)) { $cat = 'DEVICE'; - $host = $out[2] . '.' . $out[1]; $add = ' (' . $param . ')'; } elseif (preg_match('/^([a-f0-9]{4})$/', $param, $out)) { $cat = 'VENDOR'; - $host = $out[1]; } elseif (preg_match('/^c\.([a-f0-9]{2})([a-f0-9]{2})$/', $param, $out)) { $cat = 'CLASS'; - $host = $out[2] . '.' . $out[1] . '.c'; } else { die('Invalid format requested'); } - $cached = Page_Statistics::getPciId($cat, $param); - if ($cached !== false && $cached['dateline'] > time()) { - echo $cached['value'], $add; - exit; + $cached = PciId::getPciId($cat, $param, true); + if ($cached === false) { + $cached = 'Unknown'; } - $res = dns_get_record($host . '.pci.id.ucw.cz', DNS_TXT); - if (is_array($res)) { - foreach ($res as $entry) { - if (isset($entry['txt']) && substr($entry['txt'], 0, 2) === 'i=') { - $string = substr($entry['txt'], 2); - Page_Statistics::setPciId($cat, $param, $string); - echo $string, $add; - exit; - } - } - } - if ($cached !== false) { - echo $cached['value'], $add; - exit; - } - die('Not found'); - } - - public static function getPciId($cat, $id) - { - static $cache = []; - $key = $cat . '-' . $id; - if (isset($cache[$key])) - return $cache[$key]; - return $cache[$key] = Database::queryFirst('SELECT value, dateline FROM pciid WHERE category = :cat AND id = :id LIMIT 1', - array('cat' => $cat, 'id' => $id)); - } - - private static function setPciId($cat, $id, $value) - { - Database::exec('INSERT INTO pciid (category, id, value, dateline) VALUES (:cat, :id, :value, :timeout)' - . ' ON DUPLICATE KEY UPDATE value = VALUES(value), dateline = VALUES(dateline)', - array( - 'cat' => $cat, - 'id' => $id, - 'value' => $value, - 'timeout' => time() + mt_rand(10, 30) * 86400, - ), true); + echo $cached, $add; + exit; } } -- cgit v1.2.3-55-g7522