Split hardware collector and python in seperate modules and remove pip for now (maybe own module later)

6 files changed, 330 insertions, 46 deletions

diff --git a/modules.d/bas-hw-collect/module-setup.sh b/modules.d/bas-hw-collect/module-setup.sh
new file mode 100755
index 00000000..0ebccc07
--- /dev/null
+++ b/modules.d/bas-hw-collect/module-setup.sh
@@ -0,0 +1,109 @@
+#!/usr/bin/env bash
+
+check() {
+	for bin in lshw lspci ip dmidecode; do
+		if ! hash "$bin" 2>&1 /dev/null; then
+			echo "Missing '$bin' please install it..."
+			return 1
+		fi
+	done
+	return 255
+}
+depends() {
+	# drm is needed for getting the edid data
+	echo slx-drm python
+}
+install() {
+	# Copy the python hardware collection script to /opt/bas/
+	mkdir -p "$initdir/opt/bas"
+	cp -r "$moddir/scripts/collect_hw_info_json.py" "$initdir/opt/bas/"
+	cp -r "$moddir/scripts/dmiparser.py" "$initdir/opt/bas/"
+	
+	# Libs needed for python3 and dmiparser
+	mkdir -p "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3.8/__future__.py" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3.8/_collections_abc.py" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3.8/_sitebuiltins.py" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3.8/_weakrefset.py" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3.8/abc.py" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3.8/argparse.py" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3.8/base64.py" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3.8/bisect.py" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3.8/calendar.py" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3.8/codecs.py" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3.8/collections" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3.8/contextlib.py" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3.8/copy.py" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3.8/copyreg.py" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3.8/datetime.py" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3.8/email" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3.8/encodings" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3.8/enum.py" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3.8/fnmatch.py" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3.8/functools.py" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3.8/genericpath.py" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3.8/hashlib.py" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3.8/heapq.py" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3.8/hmac.py" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3.8/http" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3.8/importlib" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3.8/io.py" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3.8/json" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3.8/keyword.py" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3.8/linecache.py" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3.8/locale.py" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3.8/logging" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3.8/mimetypes.py" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3.8/operator.py" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3.8/os.py" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3.8/posixpath.py" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3.8/queue.py" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3.8/quopri.py" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3.8/random.py" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3.8/re.py" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3.8/reprlib.py" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3.8/selectors.py" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3.8/shlex.py" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3.8/shutil.py" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3.8/signal.py" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3.8/site.py" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3.8/socket.py" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3.8/sre_compile.py" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3.8/sre_constants.py" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3.8/sre_parse.py" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3.8/stat.py" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3.8/string.py" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3.8/stringprep.py" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3.8/struct.py" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3.8/subprocess.py" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3.8/threading.py" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3.8/tempfile.py" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3.8/token.py" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3.8/tokenize.py" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3.8/traceback.py" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3.8/types.py" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3.8/urllib" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3.8/uu.py" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3.8/warnings.py" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3.8/weakref.py" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3.8/zipfile.py" "$initdir/usr/lib/python3.8"
+	
+	cp -r "/usr/lib/python3/dist-packages/certifi" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3/dist-packages/chardet" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3/dist-packages/idna" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3/dist-packages/requests" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3/dist-packages/six.py" "$initdir/usr/lib/python3.8"
+	cp -r "/usr/lib/python3/dist-packages/urllib3" "$initdir/usr/lib/python3.8"
+	
+	# Certificates
+	mkdir -p "$initdir/etc/ssl/certs/"
+	cp "/etc/ssl/certs/ca-certificates.crt" "$initdir/etc/ssl/certs/"
+
+	inst_multiple lshw lspci smartctl ip dmidecode
+	
+	# Script get called via bash registration hook configured in the bas
+	# Exec the python hook and reboot instead of Switch Root
+	# inst_hook pre-mount 00 "$moddir/scripts/python_hook.sh"
+
+	return 0
+}
diff --git a/modules.d/bas-python/scripts/collect_hw_info_json.py b/modules.d/bas-hw-collect/scripts/collect_hw_info_json.py
index 14e3877a..14e3877a 100755
--- a/modules.d/bas-python/scripts/collect_hw_info_json.py
+++ b/modules.d/bas-hw-collect/scripts/collect_hw_info_json.py
diff --git a/modules.d/bas-hw-collect/scripts/dmiparser.py b/modules.d/bas-hw-collect/scripts/dmiparser.py
new file mode 100644
index 00000000..9e3a0bfa
--- /dev/null
+++ b/modules.d/bas-hw-collect/scripts/dmiparser.py
@@ -0,0 +1,201 @@
+import re
+import json
+from itertools import takewhile
+from enum import Enum
+
+__all__ = ['DmiParser']
+
+DmiParserState = Enum (
+    'DmiParserState',
+    (
+        'GET_SECT',
+        'GET_PROP',
+        'GET_PROP_ITEM',
+    )
+)
+
+class DmiParserSectionHandle(object):
+    '''A handle looks like this
+
+    Handle 0x0066, DMI type 148, 48 bytes
+    '''
+    def __init__(self):
+        self.id= ''
+        self.type = ''
+        self.bytes = 0
+
+    def __str__(self):
+        return json.dumps(self.__dict__)
+
+class DmiParserSectionProp(object):
+    '''A property looks like this
+
+    Characteristics:
+            3.3 V is provided
+            PME signal is supported
+            SMBus signal is supported
+    '''
+    def __init__(self, value:str):
+        self.values = []
+
+        if value:
+            self.append(value)
+
+    def __str__(self):
+        return json.dumps(self.__dict__)
+
+    def append(self, item:str):
+        self.values.append(item)
+
+class DmiParserSection(object):
+    '''A section looks like this
+
+    On Board Device 1 Information
+            Type: Video
+            Status: Enabled
+            Description: ServerEngines Pilot III
+    '''
+    def __init__(self):
+        self.handle = None
+        self.name = ''
+        self.props = {}
+
+    def __str__(self):
+        return json.dumps(self.__dict__)
+
+    def append(self, key:str, prop:str):
+        self.props[key] = prop
+
+class DmiParser(object):
+    '''This parse dmidecode output to JSON
+    '''
+
+    def __init__(self, text:str, **kwargs):
+        '''
+        text:   output of command dmidecode
+        kwargs: these will pass to json.dumps
+        '''
+        self._text = text
+        self._kwargs = kwargs
+        self._indentLv = lambda l: len(list(takewhile(lambda c: "\t" == c, l)))
+        self._sections = []
+
+        if type(text) is not str:
+            raise TypeError("%s want a %s but got %s" %(
+                self.__class__, type(__name__), type(text)))
+
+        self._parse(text)
+
+    def __str__(self):
+        return json.dumps(self._sections, **self._kwargs)
+
+    def _parse(self, text:str):
+        lines = self._text.splitlines()
+        rhandle = r'^Handle\s(.+?),\sDMI\stype\s(\d+?),\s(\d+?)\sbytes$'
+        section = None
+        prop = None
+        state = None
+        k, v = None, None
+
+        for i, l in enumerate(lines):
+            if i == len(lines) - 1 or DmiParserState.GET_SECT == state:
+                # Add previous section if exist
+                if section:
+                    # Add previous prop if exist
+                    if prop:
+                        section.append(k, json.loads(str(prop)))
+                        prop = None
+
+                    self._sections.append(json.loads(str(section)))
+                    section = None
+
+            if not l:
+                continue
+
+            if l.startswith('Handle'):
+                state = DmiParserState.GET_SECT
+                handle = DmiParserSectionHandle()
+                match = re.match(rhandle, l)
+                handle.id, handle.type, handle.bytes = match.groups()
+                continue
+
+            if DmiParserState.GET_SECT == state:
+                section = DmiParserSection()
+                section.handle = json.loads(str(handle))
+                section.name = l
+                state = DmiParserState.GET_PROP
+                continue
+
+            if DmiParserState.GET_PROP == state:
+                k, v = [x.strip() for x in l.split(':', 1)]
+                prop = DmiParserSectionProp(v)
+                lv = self._indentLv(l) - self._indentLv(lines[i+1])
+
+                if v:
+                    if not lv:
+                        section.append(k, json.loads(str(prop)))
+                        prop = None
+                    elif -1 == lv:
+                        state = DmiParserState.GET_PROP_ITEM
+                        continue
+                else:
+                    if -1 == lv:
+                        state = DmiParserState.GET_PROP_ITEM
+                        continue
+
+                # Next section for this handle
+                if not self._indentLv(lines[i+1]):
+                    state = DmiParserState.GET_SECT
+
+            if DmiParserState.GET_PROP_ITEM == state:
+                prop.append(l.strip())
+
+                lv = self._indentLv(l) - self._indentLv(lines[i+1])
+
+                if lv:
+                    section.append(k, json.loads(str(prop)))
+                    prop = None
+
+                    if lv > 1:
+                        state = DmiParserState.GET_SECT
+                    else:
+                        state = DmiParserState.GET_PROP
+
+if '__main__' == __name__:
+    text='''# dmidecode 3.0
+Getting SMBIOS data from sysfs.
+SMBIOS 2.7 present.
+
+Handle 0x0003, DMI type 2, 17 bytes
+Base Board Information
+	Manufacturer: Intel Corporation
+	Product Name: S2600WT2R
+	Version: H21573-372
+	Serial Number: BQWL81150522
+	Asset Tag: Base Board Asset Tag
+	Features:
+		Board is a hosting board
+		Board is replaceable
+	Location In Chassis: Part Component
+	Chassis Handle: 0x0000
+	Type: Motherboard
+	Contained Object Handles: 0
+
+    '''
+
+    # just print
+    parser = DmiParser(text)
+    #parser = DmiParser(text, sort_keys=True, indent=2)
+    print("parser is %s" %(type(parser)))
+    print(parser)
+
+    # if you want a string
+    dmistr = str(parser)
+    print("dmistr is %s" %(type(dmistr)))
+    print(dmistr)
+
+    # if you want a data structure
+    dmidata = json.loads(str(parser))
+    print("dmidata is %s" %(type(dmidata)))
+    print(dmidata)
+
diff --git a/modules.d/bas-python/module-setup.sh b/modules.d/bas-python/module-setup.sh
deleted file mode 100755
index 83f32df9..00000000
--- a/modules.d/bas-python/module-setup.sh
+++ /dev/null
@@ -1,45 +0,0 @@
-#!/usr/bin/env bash
-
-check() {
-	for bin in python3 pip3 lshw lspci ip dmidecode; do
-		if ! hash "$bin" 2>&1 /dev/null; then
-			echo "Missing '$bin' please install it..."
-			return 1
-		fi
-	done
-	return 255
-}
-depends() {
-	# drm is needed for getting the edid data
-	echo slx-drm
-}
-install() {
-	# Copy the python hardware collection script to /opt/bas/
-	mkdir -p "$initdir/opt/bas"
-	cp -r "$moddir/scripts/collect_hw_info_json.py" "$initdir/opt/bas/"
-	
-	# For testing the scripts manual, copy em all
-	# cp -r "$moddir/scripts" "$initdir/opt/bas/"
-
-	# Libs needed for python3 and dmiparser
-	mkdir -p "$initdir/usr/lib"
-	cp -r "/usr/lib/python3" "$initdir/usr/lib/"
-	cp -r "/usr/lib/python3.8" "$initdir/usr/lib/"
-	
-	mkdir -p "$initdir/usr/local/lib"
-	cp -r "/usr/local/lib/python3.8" "$initdir/usr/local/lib/"
-	
-	mkdir -p "$initdir/usr/share"
-	cp -r "/usr/share/python-wheels" "$initdir/usr/share/python-wheels/"
-	
-	# Certificates
-	mkdir -p "$initdir/etc/ssl/certs/"
-	cp "/etc/ssl/certs/ca-certificates.crt" "$initdir/etc/ssl/certs/"
-
-	inst_multiple python3 pip3 lshw lspci smartctl ip dmidecode
-	
-	# Exec the python hook and reboot instead of Switch Root
-	# inst_hook pre-mount 00 "$moddir/scripts/python_hook.sh"
-
-	return 0
-}
diff --git a/modules.d/bas-registration-hooks/module-setup.sh b/modules.d/bas-registration-hooks/module-setup.sh
index a916a4b6..e5d41424 100755
--- a/modules.d/bas-registration-hooks/module-setup.sh
+++ b/modules.d/bas-registration-hooks/module-setup.sh
@@ -10,7 +10,8 @@ check() {
 	return 255
 }
 depends() {
-	echo bas-python
+	# echo bas-python
+	echo bas-hw-collect
 }
 install() {
 	# pre-mount is the first hook with guaranteed network access
diff --git a/modules.d/python/module-setup.sh b/modules.d/python/module-setup.sh
new file mode 100755
index 00000000..05a811a7
--- /dev/null
+++ b/modules.d/python/module-setup.sh
@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+check() {
+	for bin in python3; do
+		if ! hash "$bin" 2>&1 /dev/null; then
+			echo "Missing '$bin' please install it..."
+			return 1
+		fi
+	done
+	return 255
+}
+depends() {
+	echo ""
+}
+install() {
+	inst_multiple python3
+	return 0
+}