From 65a0508a53b82e2b34266238fb26c3d067f02c80 Mon Sep 17 00:00:00 2001 From: Simon Rettberg Date: Tue, 18 Jan 2022 18:53:25 +0100 Subject: [statistics] Add comments to the HardwareParser class --- .../statistics/inc/hardwareparser.inc.php | 150 ++++++++++++++++----- 1 file changed, 114 insertions(+), 36 deletions(-) diff --git a/modules-available/statistics/inc/hardwareparser.inc.php b/modules-available/statistics/inc/hardwareparser.inc.php index 89edc8fd..952abf91 100644 --- a/modules-available/statistics/inc/hardwareparser.inc.php +++ b/modules-available/statistics/inc/hardwareparser.inc.php @@ -65,6 +65,10 @@ class HardwareParser } /** + * Turn several numeric measurements like Size, Speed, Voltage into a unitless + * base representation, meant for comparison. For example, Voltages are converted + * to Millivolts, Anything measured in [KMGT]Bytes (per second) to bytes, GHz to + * Hz, and so on. * @param string $key * @param string $value * @return ?int value, or null if not numeric @@ -231,6 +235,15 @@ class HardwareParser } } + /** + * Insert some hardware into database. $global is supposed to contain key-value-pairs of properties + * this hardware has that is the same for every instance of this hardware, like model number, speed + * or size. Individual properties, like a serial number, are considered local properties, and go + * into a different table, that would contain a row for each client that has this hardware. + * @param string $dbType Hardware typ (HDD, RAM, ...) + * @param array $global associative array of properties this hardware has + * @return int id of this hardware; primary key of row in statistic_hw_prop + */ private static function writeGlobalHardwareData(string $dbType, array $global): int { static $cache = []; @@ -240,7 +253,7 @@ class HardwareParser $id = md5(implode(' ', $global)); if (!isset($cache[$id])) { // Cache lookup, make sure we insert this only once for every run, as this is supposed to be general - // information about the hardware, e.g. model number, max. resultion, capacity, ... + // information about the hardware, e.g. model number, max. resolution, capacity, ... $hwid = Database::insertIgnore('statistic_hw', 'hwid', ['hwtype' => $dbType, 'hwname' => $id]); $vals = []; foreach ($global as $k => $v) { @@ -257,6 +270,12 @@ class HardwareParser return $cache[$id]; } + /** + * Process hardware info for given client. + * @param string $uuid System-UUID of client + * @param array $data Hardware info, deserialized assoc array. + * @return void + */ public static function parseMachine(string $uuid, array $data) { $version = $data['version'] ?? 0; @@ -269,46 +288,63 @@ class HardwareParser $localMainboardExtra = []; // physical memory array $memArrays = self::getDmiHandles($data, 16); + // We mostly have a seprate hardware type for all the dmi types, but not for memory arrays. + // These get added to the mainboard hw-type as it's practically a property of the mainboard. + // While we can have multiple physical memory arrays, we only ever have one mainboard per + // client. Add up the data from all arrays. $globalMainboardExtra['Memory Slot Count'] = 0; $globalMainboardExtra['Memory Maximum Capacity'] = 0; foreach ($memArrays as $mem) { $mem = self::prepareDmiProperties($mem); - if (isset($mem['Number Of Devices']) && ($mem['Use'] ?? 0) === 'System Memory') { + // Not all memory arrays are for RAM.... + if (($mem['Use'] ?? 0) !== 'System Memory') + continue; + if (isset($mem['Number Of Devices'])) { $globalMainboardExtra['Memory Slot Count'] += $mem['Number Of Devices']; } if (isset($mem['Maximum Capacity'])) { + // Temporary unit is MB $globalMainboardExtra['Memory Maximum Capacity'] += self::convertSize($mem['Maximum Capacity'], 'M', false); } } + // Now finally convert to GB $globalMainboardExtra['Memory Maximum Capacity'] = self::convertSize($globalMainboardExtra['Memory Maximum Capacity'] . ' MB', 'G'); - // BIOS section - need to cross-match this with mainboard or system model, as it doesn't have a meaningful - // identifier on its own + // BIOS section - need to combine this with mainboard or system model, as it doesn't have a meaningful + // identifier on its own. So again like above, we add this to the mainboard data. $bios = self::prepareDmiProperties(self::getDmiHandles($data, 0)[0]); foreach (['Version', 'Release Date', 'Firmware Revision'] as $k) { if (isset($bios[$k])) { + // Prefix with "BIOS" to clarify, since it's added to the mainboard meta-data $localMainboardExtra['BIOS ' . $k] = $bios[$k]; } } - if (isset($bios['BIOS Revision'])) { + if (isset($bios['BIOS Revision'])) { // This one already has the BIOS prefix $localMainboardExtra['BIOS Revision'] = $bios['BIOS Revision']; } + // Vendor and ROM size of BIOS *should* always be the same for a specific mainboard foreach (['Vendor', 'ROM Size'] as $k) { if (isset($bios[$k])) { $globalMainboardExtra['BIOS ' . $k] = $bios[$k]; } } - // Using the general helper function + // "Normal" dmi entries - these map directly to one of our hardware types + // RAM modules $capa = 0; - $ramModCount = self::updateHwTypeFromDmi($uuid, $data, 17, HardwareInfo::RAM_MODULE, function (array $flat) use (&$capa): bool { - $size = self::convertSize(($flat['Size'] ?? 0), '', false); - if ($size > 65 * 1024 * 1024) { - $capa += $size; - return true; - } - return false; - }, + $ramModCount = self::updateHwTypeFromDmi($uuid, $data, 17, HardwareInfo::RAM_MODULE, + // Filter callback - we can modify the entry, or return false to ignore it + function (array $flat) use (&$capa): bool { + $size = self::convertSize(($flat['Size'] ?? 0), '', false); + // Let's assume we're never running on old HW with <=128MB modules, so this + // might be a hint that this is some other kind of memory. The proper way would be + // to check if the related physical memory array (16) has "Use" = "System Memory" + if ($size > 129 * 1024 * 1024) { + $capa += $size; + return true; + } + return false; + }, ['Locator'], ['Data Width', 'Size', @@ -322,25 +358,33 @@ class HardwareParser 'Maximum Voltage'], ['Locator', 'Bank Locator', 'Serial Number', 'Asset Tag', 'Configured Memory Speed', 'Configured Voltage'] ); - // Fake RAM slots used/total etc. into this + // Put RAM slots used/total etc. into mainboard data $localMainboardExtra['Memory Slot Occupied'] = $ramModCount; $localMainboardExtra['Memory Installed Capacity'] = self::convertSize($capa, 'G', true); + // Also add generic socket, core and thread count to mainboard data. This doesn't seem to make too much sense + //at first since it's not a property of the mainboard. But we can get away with it since we make it a local + // property, i.e. specific to a client. This is just aggregated, so it's not super well suited for the CPU + // hardware type, referenced below. In fact, on some systems the dmi/smbios tables don't contain all that much + // information about the CPU at all, so we have at least this. foreach (['sockets', 'cores', 'threads'] as $key) { if (!isset($data['cpu'][$key])) continue; $localMainboardExtra['cpu-' . $key] = $data['cpu'][$key]; } + // Finally handle mainbard data, with all of our gathered extra fields self::updateHwTypeFromDmi($uuid, $data, 2, HardwareInfo::MAINBOARD, ['Manufacturer', 'Product Name'], [], ['Manufacturer', 'Product Name', 'Type', 'Version'], ['Serial Number', 'Asset Tag', 'Location In Chassis'], $globalMainboardExtra, $localMainboardExtra ); + // System information, mostly set by OEMs, might be empty/bogus on custom systems self::updateHwTypeFromDmi($uuid, $data, 1, HardwareInfo::DMI_SYSTEM, ['Manufacturer', 'Product Name'], [], ['Manufacturer', 'Product Name', 'Version', 'Wake-up Type'], ['Serial Number', 'UUID', 'SKU Number'] ); + // Might contain more or less accurate information. Mostly works on servers and OEM systems self::updateHwTypeFromDmi($uuid, $data, 39, HardwareInfo::POWER_SUPPLY, ['Manufacturer'], ['Location', 'Power Unit Group', @@ -348,32 +392,35 @@ class HardwareParser ['Manufacturer', 'Product Name', 'Model Part Number', 'Revision', 'Max Power Capacity'], ['Serial Number', 'Asset Tag', 'Status', 'Plugged', 'Hot Replaceable'] ); + // On some more recent systems this contains quite some useful information self::updateHwTypeFromDmi($uuid, $data, 4, HardwareInfo::CPU, ['Version'], ['Socket Designation'], ['Type', 'Family', 'Manufacturer', 'Signature', 'Version', 'Core Count', 'Thread Count'], ['Voltage', 'Current Speed', 'Upgrade', 'Core Enabled']); // Information about system slots - self::updateHwTypeFromDmi($uuid, $data, 9, HardwareInfo::SYSTEM_SLOT, function (array &$entry): bool { - // Use a callback filter to extract PCIe slot metadata into unique fields - if (!isset($entry['Type'])) - return false; - // Split up PCIe info - if (preg_match('/^x(?\d+) PCI Express( (?\d+)( x(?\d+))?)?$/', $entry['Type'], $out)) { - $entry['Type'] = 'PCI Express'; - $entry['PCIe Bus Width'] = $out['b']; - if (!empty($out['g'])) { - $entry['PCIe Gen'] = $out['g']; - } - if (!empty($out['s'])) { - $entry['PCIe Slot Width'] = $out['s']; + self::updateHwTypeFromDmi($uuid, $data, 9, HardwareInfo::SYSTEM_SLOT, + function (array &$entry): bool { + // Use a callback filter to extract PCIe slot metadata into unique fields + if (!isset($entry['Type'])) + return false; + // Split up PCIe info – gen, electrical width and physical width are mashed into one field + if (preg_match('/^x(?\d+) PCI Express( (?\d+)( x(?\d+))?)?$/', $entry['Type'], $out)) { + $entry['Type'] = 'PCI Express'; + $entry['PCIe Bus Width'] = $out['b']; + if (!empty($out['g'])) { + $entry['PCIe Gen'] = $out['g']; + } + if (!empty($out['s'])) { + $entry['PCIe Slot Width'] = $out['s']; + } } - } - return true; - }, + return true; + }, ['Designation', 'ID', 'Bus Address'], ['Type', 'PCIe Bus Width', 'PCIe Gen', 'PCIe Slot Width'], ['Current Usage', 'Designation'] ); + // dmidecode end // ---- lspci ------------------------------------ $pciHwIds = []; foreach (($data['lspci'] ?? []) as $dev) { @@ -386,12 +433,14 @@ class HardwareParser $pciHwIds[] = $mappingId; } self::markDisconnected($uuid, HardwareInfo::PCI_DEVICE, $pciHwIds); - // ---- Disks ------------------------------------0Y3R3K + // ---- Disks ------------------------------------ $hddHwIds = []; + // Sum of all ID44/45 partitions in bytes $id44 = $id45 = 0; foreach (($data['drives'] ?? []) as $dev) { - if (empty($dev['readlink'])) + if (empty($dev['readlink'])) // This is the canonical entry name directly under /dev/, e.g. /dev/sda continue; + // Use smartctl as the source of truth, lsblk as fallback if data is missing if (!isset($dev['smartctl'])) { $smart = []; } else { @@ -403,6 +452,7 @@ class HardwareParser $lsblk =& $dev['lsblk']['blockdevices'][0]; } if (!isset($smart['rotation_rate']) && isset($lsblk['rota']) && !$lsblk['rota']) { + // smartctl didn't report on it, lsblk says it's non-rotational $smart['rotation_rate'] = 0; } $size = $lsblk['size'] ?? $smart['user_capacity']['bytes'] ?? 'unknown'; @@ -447,7 +497,7 @@ class HardwareParser return !is_array($v) && $k !== 'temperature' && $k !== 'temperature_sensors'; }, ARRAY_FILTER_USE_BOTH); } - // Partitions - find special ones + // Partitions $used = 0; if (isset($dev['sfdisk']['partitiontable'])) { $table['partition_table'] = $dev['sfdisk']['partitiontable']['label'] ?? 'none'; @@ -467,8 +517,10 @@ class HardwareParser continue; $type = hexdec($part['type'] ?? '0'); if ($type === 0x0 || $type === 0x5 || $type === 0xf || $type === 0x15 || $type === 0x1f - || $type === 0x85 || $type === 0xc5 || $type == 0xcf) - continue; // Extended partition, ignore + || $type === 0x85 || $type === 0xc5 || $type == 0xcf) { + // Extended partition, ignore + continue; + } $used += $part['size'] * $fac; $id = 'part_' . $i . '_'; foreach (['start', 'size', 'type', 'uuid', 'name'] as $item) { @@ -566,6 +618,10 @@ class HardwareParser /** * Takes key-value-array, returns a new array with only the keys listed in $fields. + * Checks if the given key is not an array. If it's an array, it will be ignored. + * Supports nested arrays. Nested keys are separated by '//', so to query + * $array['x']['y'], add 'x//y' to $fields. The value will be added to the return + * value as key 'x//y'. */ private static function propsFromArray(array $array, string ...$fields): array { @@ -594,6 +650,28 @@ class HardwareParser return $ret; } + /** + * Extract data from dmi/smbios and write to DB. + * This is a pretty involved function that does several things, among them is splitting + * up the data by global and local properties, create new hardware entry if new, and + * making sure other hardware of same type gets marked as disconnected from given client. + * @param string $uuid client uuid + * @param array $data dmidecode part of hardware info + * @param int $type dmi type to extract from $data + * @param string $dbType hardware type to write this to DB as + * @param array|callable $requiredPropsOrCallback either a list of properties that are + * mandatory for this hwtype, or a callback function that returns true/false for + * valid/invalid dmi entries + * @param array $pathFields fields from entry that define the path or location of the + * hardware in the client + * @param array $globalProps properties of entry that are considered the same for all + * instances of that hardware, e.g. model name + * @param array $localProps properties of entry that are considered different for each + * instance of this hardware per client, e.g. serial number, power-on hours + * @param array $globalExtra additional key-value-pairs to write to DB as being global + * @param array $localExtra additional key-value-pairs to write to DB as being local + * @return int number of table entries written to DB, i.e. passed $requiredPropsOrCallback + */ private static function updateHwTypeFromDmi( string $uuid, array $data, int $type, string $dbType, $requiredPropsOrCallback, array $pathFields, array $globalProps, array $localProps, -- cgit v1.2.3-55-g7522