diff options
3 files changed, 588 insertions, 8 deletions
diff --git a/core/modules/hardware-stats/data/opt/openslx/hardware-stats/collect_hw_info_json.py b/core/modules/hardware-stats/data/opt/openslx/hardware-stats/collect_hw_info_json.py new file mode 100644 index 00000000..95f96378 --- /dev/null +++ b/core/modules/hardware-stats/data/opt/openslx/hardware-stats/collect_hw_info_json.py @@ -0,0 +1,386 @@ +from dmiparser import DmiParser +from subprocess import PIPE +from os import listdir, path +import argparse +import json +import re +import requests +import shlex +import subprocess +import sys + +__debug = False + +# Run dmi command as subprocess and get the stdout +def run_subprocess(_cmd): + global __debug + proc = subprocess.run(_cmd, shell=True, stdout=PIPE, stderr=PIPE) + stdout = proc.stdout + stderr = proc.stderr + + if __debug: + eprint(_cmd + ':') + eprint() + eprint('Errors:') + eprint(stderr.decode()) + eprint() + # stderr len instead of proc.returncode > 0 is used because some have returncode 2 but are still valid + if len(stderr.decode()) > 0: + eprint(_cmd + ' Errors:') + eprint(stderr.decode()) + if proc.returncode != 0: + eprint('Error Return Code: ' + str(proc.returncode)) + eprint() + return False + else: + return stdout.decode() + else: + return stdout.decode() + +# Get and parse dmidecode using dmiparser +def get_dmidecode(): + _dmiraw = run_subprocess('dmidecode') + _dmiparsed = '' + if _dmiraw: + # Parse dmidecode + _dmiparsed = DmiParser(_dmiraw) + return json.loads(str(_dmiparsed)) + else: + return [] + +# Get the readlink -f output +def get_readlink(link): + _readlink = run_subprocess('readlink -f ' + link) + return _readlink + +# Try parsing string to json +def parse_to_json(program_name, raw_string): + parsed = {} + if isinstance(raw_string, str): + try: + parsed = json.loads(raw_string) + except ValueError as e: + eprint(program_name + ' output couldn\'t be parsed') + eprint(e) + eprint('Output of ' + program_name + 'was:') + eprint(raw_string) + eprint() + return parsed + +# Get smartctl output in json format +def get_smartctl(disk_path, disk_name): + disk_path_full = disk_path + disk_name + output = run_subprocess('smartctl -x --json ' + disk_path_full) + smartctl = parse_to_json('smartctl', output) + return smartctl + +# Get sfdisk info in json format +def get_sfdisk(disk_path, disk_name): + output = run_subprocess('sfdisk --json ' + disk_path + disk_name) + sfdisk = parse_to_json('sfdisk', output) + return sfdisk + +# Get lsblk info in json format +def get_lsblk(disk_path, disk_name): + output = run_subprocess('lsblk --json -b --output-all ' + disk_path + disk_name) + lsblk = parse_to_json('lsblk', output) + return lsblk + +# Get CD/DVD Information +def get_cdrom(): + cdromdir = '/proc/sys/dev/cdrom/' + cdrom = [] + if path.exists(cdromdir): + cdrom_raw = run_subprocess('cat ' + cdromdir + 'info') + + # Skip first two entries because of useless information and empty row + for row in cdrom_raw.split('\n')[2:]: + if row == '': + continue + # Split at one or more tabs + values = re.split('\t+', row) + key = values[0][:-1].replace('drive ', '').replace(' ', '_') + for index, val in enumerate(values[1:]): + if len(cdrom) < index + 1: + cdrom.append({ 'read': [], 'write': [], 'functions': [] }) + + if 'Can_read_' in key: + if val == '1': + cdrom[index]['read'].append(key[9:]) + elif 'Can_write_' in key: + cdrom[index]['write'].append(key[10:]) + elif 'Can_' in key: + cdrom[index]['functions'].append(key[4:]) + else: + cdrom[index][key] = val + return {cd['name']:cd for cd in cdrom} + +# Get informations about the disks +def get_disk_info(): + diskdir = '/dev/disk/by-path/' + disk_informations = {} + dupcheck = {} + cdrom = get_cdrom() + + # Get and filter all disks + if path.exists(diskdir): + disks = listdir(diskdir) + filtered_disks = [i for i in disks if (not '-part' in i) and (not '-usb-' in i)] + + # Call all disk specific tools + for d in filtered_disks: + disk_path = diskdir + d + devpath = get_readlink(disk_path).rstrip() + # Sometimes there are multiple links to the same disk, e.g. named + # pci-0000:00:1f.2-ata-1.0 and pci-0000:00:1f.2-ata-1 + if devpath in dupcheck: + continue + dupcheck[devpath] = 1 + disk_info = {} + disk_info['readlink'] = devpath + + # Check if it's a cd/dvd + if disk_info['readlink'].split('/')[-1] in cdrom.keys(): + disk_info['type'] = 'cdrom' + disk_info['info'] = cdrom[disk_info['readlink'].split('/')[-1]] + else: + disk_info['type'] = 'drive' + disk_info['smartctl'] = get_smartctl(diskdir, d) + disk_info['lsblk'] = get_lsblk(diskdir, d) + + if disk_info['type'] != 'cdrom': + disk_info['sfdisk'] = get_sfdisk(diskdir, d) + + disk_informations[d] = disk_info + return disk_informations + +# Get and process "lspci -mn" output +def get_lspci(): + lspci = [] + lspci_raw = run_subprocess('lspci -mmn').split('\n') + + # Prepare addition of iommu group + iommu_groups = {} + iommu_raw = run_subprocess('find /sys/kernel/iommu_groups/*/devices/*') + if iommu_raw: + iommu_split = iommu_raw.split('\n') + for iommu_path in iommu_split: + if iommu_path == "": + continue + iommu = iommu_path.split('/') + iommu_groups[iommu[6][5:]] = iommu[4] + + for line in lspci_raw: + if len(line) <= 0: continue + + # Parsing shell like command parameters + parse = shlex.split(line) + lspci_parsed = {} + arguments = [] + values = [] + for parameter in parse: + # Split values from arguments + if parameter.startswith('-'): + arguments.append(parameter) + else: + values.append(parameter) + + # Prepare values positions are in order + if len(values) >= 6: + lspci_parsed['slot'] = values[0] + + # The busybox version of lspci has "Class <class>" instead of "<class>" + lspci_parsed['class'] = values[1].replace("Class ", "") + lspci_parsed['vendor'] = values[2] + lspci_parsed['device'] = values[3] + lspci_parsed['subsystem_vendor'] = values[4] + lspci_parsed['subsystem'] = values[5] + + # Additional add iommu group + if values[0] in iommu_groups: + lspci_parsed['iommu_group'] = iommu_groups[values[0]] + + # Prepare arguments + if len(arguments) > 0: + for arg in arguments: + if arg.startswith('-p'): + lspci_parsed['progif'] = arg[2:] + elif arg.startswith('-r'): + lspci_parsed['rev'] = arg[2:] + else: continue + + lspci.append(lspci_parsed) + return lspci + +# Get ip data in json format +def get_ip(): + result = [] + ip_raw = run_subprocess('ip --json addr show') + if isinstance(ip_raw, str): + result = json.loads(ip_raw) + return result + +def get_net_fallback(): + netdir = '/sys/class/net/' + result = {} + + # Get MAC address and speed + if path.exists(netdir): + interfaces = run_subprocess('ls ' + netdir).split('\n') + for interface in interfaces: + # Skip local stuff + if interface == 'lo' or interface == '': + continue + net = {} + speed = run_subprocess('cat ' + netdir + interface + '/speed') + if isinstance(speed, str) and speed.endswith('\n'): + net['speed'] = speed[:-1] + + duplex = run_subprocess('cat ' + netdir + interface + '/duplex') + if isinstance(duplex, str) and duplex.endswith('\n'): + net['duplex'] = duplex[:-1] + + mac = run_subprocess('cat ' + netdir + interface + '/address') + if isinstance(mac, str) and mac.endswith('\n'): + net['mac'] = mac[:-1] + result[interface] = net + + # Get IP address + interfaces = run_subprocess('ip -o addr show | awk \'/inet/ {print $2, $3, $4}\'').split('\n') + for interface in interfaces: + if interface == '': + continue + interf = interface.split(' ') + if interf[0] == 'lo': + continue + else: + if interf[0] not in result: + result[interf[0]] = {} + if interf[1] == 'inet': + result[interf[0]]['ipv4'] = interf[2] + elif interf[1] == 'inet6': + result[interf[0]]['ipv6'] = interf[2] + return result + +# Get and convert EDID data to hex +def get_edid(): + edid = {} + display_paths = run_subprocess('ls /sys/class/drm/*/edid') + if display_paths: + display_paths = display_paths.split('\n') + for dp in display_paths: + if dp == '': continue + edid_hex = open(dp, 'rb').read().hex() + if len(edid_hex) > 0: + # The path is always /sys/class/drm/[..]/edid, so cut the first 15 chars and the last 5 chars + edid[dp[15:-5]] = edid_hex + return edid + +def get_lshw(): + result = [] + lshw_raw = run_subprocess('lshw -json') + if isinstance(lshw_raw, str): + result = json.loads(lshw_raw) + return result + +def get_uuid(): + uuid_path = '/sys/class/dmi/id/product_uuid' + uuid = 'N/A' + if path.exists(uuid_path): + uuid_raw = run_subprocess('cat ' + uuid_path) + # uuid_raw = False if no sudo permission: + if uuid_raw: + uuid = uuid_raw.rstrip() + return uuid + +def prepare_location(parent, bay, slot): + location = {} + if parent: + location['parent'] = parent + if bay: + location['bay'] = int(bay) + if slot: + location['slot'] = int(slot) + return location + +def prepare_contacts(contact_list): + contacts = [] + if contact_list == None: + return contacts + for contact in contact_list: + contacts.append(contact[0]) + return contacts + +def send_post(url, payload): + # headers = { 'Content-type': 'application/json', 'Accept': 'text/plain' } + # req = requests.post(url, json=payload, headers=headers) + req = requests.post(url, json=payload) + # Print the response + print("POST-Request Response: \n") + print(req.text) + +def eprint(*args, **kwargs): + print(*args, file=sys.stderr, **kwargs) + +def main(): + global __debug + + # Create and parse arguments + parser = argparse.ArgumentParser(description='Collects hardware data from different tools and returns it as json.') + parser.add_argument('-d', '--debug', action='store_true', help='Prints all STDERR messages. (Non critical included)') + parser.add_argument('-u', '--url', action='append', help='[multiple] If given, a post request with the generated JSON is sent to the given URLs') + parser.add_argument('-uu', '--uuidurl', action='append', help='[multiple] Same as -u but UUID in the url is replaced with the actual uuid of the client') + parser.add_argument('-p', '--print', action='store_true', help='Prints the generated JSON') + parser.add_argument('-l', '--location', action='store', help='Room-/Rackname where the client is located') + parser.add_argument('-s', '--slot', action='store', help='The slot number (int) where the client is located in the rack') + parser.add_argument('-b', '--bay', action='store', help='The bay number (int) where the client is located in the slot (segment)') + parser.add_argument('-c', '--contact', action='append', help='[multiple] The idoit_username of the person responsible for this machine', nargs=1) + parser.add_argument('-n', '--name', action='store', help='Name of the client') + parser.add_argument('-S', '--SERVER', action='store_true', help='Defines the type of the client to be a server') + args = parser.parse_args() + + if args.debug: + __debug = True + + # Run the tools + _collecthw = {} + _collecthw['version'] = 2.0 + _collecthw['dmidecode'] = get_dmidecode() + + # Includes smartctl, lsblk, readlink and sfdisk + _collecthw['drives'] = get_disk_info() + + # Includes iommu group + _collecthw['lspci'] = get_lspci() + + _collecthw['ip'] = get_ip() + _collecthw['edid'] = get_edid() + _collecthw['lshw'] = get_lshw() + _collecthw['net'] = get_net_fallback() + + _collecthw['location'] = prepare_location(args.location, args.bay, args.slot) + _collecthw['contacts'] = prepare_contacts(args.contact) + + if args.name: + _collecthw['name'] = args.name + + if args.SERVER: + _collecthw['type'] = 'SERVER' + + collecthw_json = json.dumps(_collecthw) + if args.url: + for url in args.url: + send_post(url, _collecthw) + if args.uuidurl: + uuid = get_uuid() + for uuidurl in args.uuidurl: + url = uuidurl.replace("UUID", uuid) + send_post(url, _collecthw) + + # Print out the final json + if args.print: + print(json.dumps(_collecthw)) + +if __name__ == "__main__": + main() + diff --git a/core/modules/hardware-stats/data/opt/openslx/hardware-stats/dmiparser.py b/core/modules/hardware-stats/data/opt/openslx/hardware-stats/dmiparser.py new file mode 100644 index 00000000..9e3a0bfa --- /dev/null +++ b/core/modules/hardware-stats/data/opt/openslx/hardware-stats/dmiparser.py @@ -0,0 +1,201 @@ +import re +import json +from itertools import takewhile +from enum import Enum + +__all__ = ['DmiParser'] + +DmiParserState = Enum ( + 'DmiParserState', + ( + 'GET_SECT', + 'GET_PROP', + 'GET_PROP_ITEM', + ) +) + +class DmiParserSectionHandle(object): + '''A handle looks like this + + Handle 0x0066, DMI type 148, 48 bytes + ''' + def __init__(self): + self.id= '' + self.type = '' + self.bytes = 0 + + def __str__(self): + return json.dumps(self.__dict__) + +class DmiParserSectionProp(object): + '''A property looks like this + + Characteristics: + 3.3 V is provided + PME signal is supported + SMBus signal is supported + ''' + def __init__(self, value:str): + self.values = [] + + if value: + self.append(value) + + def __str__(self): + return json.dumps(self.__dict__) + + def append(self, item:str): + self.values.append(item) + +class DmiParserSection(object): + '''A section looks like this + + On Board Device 1 Information + Type: Video + Status: Enabled + Description: ServerEngines Pilot III + ''' + def __init__(self): + self.handle = None + self.name = '' + self.props = {} + + def __str__(self): + return json.dumps(self.__dict__) + + def append(self, key:str, prop:str): + self.props[key] = prop + +class DmiParser(object): + '''This parse dmidecode output to JSON + ''' + + def __init__(self, text:str, **kwargs): + ''' + text: output of command dmidecode + kwargs: these will pass to json.dumps + ''' + self._text = text + self._kwargs = kwargs + self._indentLv = lambda l: len(list(takewhile(lambda c: "\t" == c, l))) + self._sections = [] + + if type(text) is not str: + raise TypeError("%s want a %s but got %s" %( + self.__class__, type(__name__), type(text))) + + self._parse(text) + + def __str__(self): + return json.dumps(self._sections, **self._kwargs) + + def _parse(self, text:str): + lines = self._text.splitlines() + rhandle = r'^Handle\s(.+?),\sDMI\stype\s(\d+?),\s(\d+?)\sbytes$' + section = None + prop = None + state = None + k, v = None, None + + for i, l in enumerate(lines): + if i == len(lines) - 1 or DmiParserState.GET_SECT == state: + # Add previous section if exist + if section: + # Add previous prop if exist + if prop: + section.append(k, json.loads(str(prop))) + prop = None + + self._sections.append(json.loads(str(section))) + section = None + + if not l: + continue + + if l.startswith('Handle'): + state = DmiParserState.GET_SECT + handle = DmiParserSectionHandle() + match = re.match(rhandle, l) + handle.id, handle.type, handle.bytes = match.groups() + continue + + if DmiParserState.GET_SECT == state: + section = DmiParserSection() + section.handle = json.loads(str(handle)) + section.name = l + state = DmiParserState.GET_PROP + continue + + if DmiParserState.GET_PROP == state: + k, v = [x.strip() for x in l.split(':', 1)] + prop = DmiParserSectionProp(v) + lv = self._indentLv(l) - self._indentLv(lines[i+1]) + + if v: + if not lv: + section.append(k, json.loads(str(prop))) + prop = None + elif -1 == lv: + state = DmiParserState.GET_PROP_ITEM + continue + else: + if -1 == lv: + state = DmiParserState.GET_PROP_ITEM + continue + + # Next section for this handle + if not self._indentLv(lines[i+1]): + state = DmiParserState.GET_SECT + + if DmiParserState.GET_PROP_ITEM == state: + prop.append(l.strip()) + + lv = self._indentLv(l) - self._indentLv(lines[i+1]) + + if lv: + section.append(k, json.loads(str(prop))) + prop = None + + if lv > 1: + state = DmiParserState.GET_SECT + else: + state = DmiParserState.GET_PROP + +if '__main__' == __name__: + text='''# dmidecode 3.0 +Getting SMBIOS data from sysfs. +SMBIOS 2.7 present. + +Handle 0x0003, DMI type 2, 17 bytes +Base Board Information + Manufacturer: Intel Corporation + Product Name: S2600WT2R + Version: H21573-372 + Serial Number: BQWL81150522 + Asset Tag: Base Board Asset Tag + Features: + Board is a hosting board + Board is replaceable + Location In Chassis: Part Component + Chassis Handle: 0x0000 + Type: Motherboard + Contained Object Handles: 0 + + ''' + + # just print + parser = DmiParser(text) + #parser = DmiParser(text, sort_keys=True, indent=2) + print("parser is %s" %(type(parser))) + print(parser) + + # if you want a string + dmistr = str(parser) + print("dmistr is %s" %(type(dmistr))) + print(dmistr) + + # if you want a data structure + dmidata = json.loads(str(parser)) + print("dmidata is %s" %(type(dmidata))) + print(dmidata) + diff --git a/core/modules/hardware-stats/module.build b/core/modules/hardware-stats/module.build index c9104ab7..3c6eb632 100644 --- a/core/modules/hardware-stats/module.build +++ b/core/modules/hardware-stats/module.build @@ -4,14 +4,7 @@ fetch_source() { } build() { - local dir="${MODULE_BUILD_DIR}/opt/openslx/hardware-stats" - mkdir -p "$dir" - cde "$dir" - local file - for file in collect_hw_info_json.py dmiparser.py; do - wget -O "$file" "https://git.openslx.org/openslx-ng/systemd-init.git/plain/modules.d/bas-hw-collect/scripts/${file}?h=bas" \ - || perror "Could not download $file" - done + : } post_copy() { |