From f2133e98aa5457f99e4ab7e8f8ac48b9720fb777 Mon Sep 17 00:00:00 2001
From: Simon Rettberg
Date: Mon, 22 Aug 2016 12:11:27 +0200
Subject: Modularize cron api

Modules now can have a ./hooks/cron.inc.php to run every X minutes,
according to how you set up the cron job for the global /inc/cron.inc.php
---
 modules-available/eventlog/hooks/cron.inc.php   |  5 +++++
 modules-available/statistics/hooks/cron.inc.php | 18 ++++++++++++++++++
 modules-available/syslog/hooks/cron.inc.php     |  5 +++++
 3 files changed, 28 insertions(+)
 create mode 100644 modules-available/eventlog/hooks/cron.inc.php
 create mode 100644 modules-available/statistics/hooks/cron.inc.php
 create mode 100644 modules-available/syslog/hooks/cron.inc.php

(limited to 'modules-available')

diff --git a/modules-available/eventlog/hooks/cron.inc.php b/modules-available/eventlog/hooks/cron.inc.php
new file mode 100644
index 00000000..027acf87
--- /dev/null
+++ b/modules-available/eventlog/hooks/cron.inc.php
@@ -0,0 +1,5 @@
+<?php
+
+if (mt_rand(1, 10) === 1) {
+	Database::exec("DELETE FROM eventlog WHERE (UNIX_TIMESTAMP() - dateline) > 86400 * 190");
+}
\ No newline at end of file
diff --git a/modules-available/statistics/hooks/cron.inc.php b/modules-available/statistics/hooks/cron.inc.php
new file mode 100644
index 00000000..94c65248
--- /dev/null
+++ b/modules-available/statistics/hooks/cron.inc.php
@@ -0,0 +1,18 @@
+<?php
+
+Database::exec("DELETE FROM statistic WHERE (UNIX_TIMESTAMP() - dateline) > 86400 * 190");
+Database::exec("DELETE FROM machine WHERE (UNIX_TIMESTAMP() - lastseen) > 86400 * 365");
+
+function logstats() {
+	$NOW = time();
+	$cutoff = $NOW - 86400 * 30;
+	$online = $NOW - 610;
+	$known = Database::queryFirst("SELECT Count(*) AS val FROM machine WHERE lastseen > $cutoff");
+	$on = Database::queryFirst("SELECT Count(*) AS val FROM machine WHERE lastseen > $online");
+	$used = Database::queryFirst("SELECT Count(*) AS val FROM machine WHERE lastseen > $online AND logintime <> 0");
+	Database::exec("INSERT INTO statistic (dateline, typeid, clientip, username, data) VALUES (:now, '~stats', '', '', :vals)", array(
+		'now' => $NOW,
+		'vals' => $known['val'] . '#' . $on['val'] . '#' . $used['val'],
+	));
+}
+logstats();
diff --git a/modules-available/syslog/hooks/cron.inc.php b/modules-available/syslog/hooks/cron.inc.php
new file mode 100644
index 00000000..c796675f
--- /dev/null
+++ b/modules-available/syslog/hooks/cron.inc.php
@@ -0,0 +1,5 @@
+<?php
+
+if (mt_rand(1, 10) === 1) {
+	Database::exec("DELETE FROM clientlog WHERE (UNIX_TIMESTAMP() - dateline) > 86400 * 190");
+}
\ No newline at end of file
-- 
cgit v1.2.3-55-g7522


From 9192cb92901349a0ddb6940b4f0ebe6110f3b9ab Mon Sep 17 00:00:00 2001
From: Simon Rettberg
Date: Mon, 22 Aug 2016 12:12:30 +0200
Subject: [statistics] Handle lastboot = 0 as offline, fix hdd parsing, add raw
 display

---
 modules-available/statistics/inc/parser.inc.php |  4 +++-
 modules-available/statistics/page.inc.php       | 28 +++++++++++++++++--------
 2 files changed, 22 insertions(+), 10 deletions(-)

(limited to 'modules-available')

diff --git a/modules-available/statistics/inc/parser.inc.php b/modules-available/statistics/inc/parser.inc.php
index 1ecfbbf8..07bed8e5 100644
--- a/modules-available/statistics/inc/parser.inc.php
+++ b/modules-available/statistics/inc/parser.inc.php
@@ -103,6 +103,8 @@ class Parser {
 		$i = 0;
 		foreach ($lines as $line) {
 			if (preg_match('/^Disk (\S+):.* (\d+) bytes/i', $line, $out)) {
+				if ($out[2] < 10000) // sometimes vmware reports lots of 512byte disks
+					continue;
 				// disk total size and name
 				unset($hdd);
 				$unit = 0;
@@ -144,7 +146,7 @@ class Parser {
 		foreach ($hdds as &$hdd) {
 			$hdd['used'] = round($hdd['used'] / 1024);
 			$free = $hdd['size'] - $hdd['used'];
-			if ($free > 5) {
+			if ($free > 5 || ($free / $hdd['size']) > 0.1) {
 				$hdd['partitions'][] = array(
 					'id' => 'free-id-' . $i,
 					'name' => Dictionary::translate('unused'),
diff --git a/modules-available/statistics/page.inc.php b/modules-available/statistics/page.inc.php
index ab0afcea..0639d784 100644
--- a/modules-available/statistics/page.inc.php
+++ b/modules-available/statistics/page.inc.php
@@ -136,6 +136,8 @@ class Page_Statistics extends Page
 			Message::addSuccess('notes-saved');
 			Util::redirect('?do=Statistics&uuid=' . $uuid);
 		}
+		// Fix online state of machines that crashed -- TODO: Make cronjob for this
+		Database::exec("UPDATE machine SET lastboot = 0 WHERE lastseen < UNIX_TIMESTAMP() - 610");
 	}
 
 	protected function doRender()
@@ -253,10 +255,9 @@ class Page_Statistics extends Page
 	{
 		$filterSet->makeFragments($where, $join, $sort, $args);
 
-		$online = time() - 610;
 		$known = Database::queryFirst("SELECT Count(*) AS val FROM machine $join WHERE ($where)", $args);
-		$on = Database::queryFirst("SELECT Count(*) AS val FROM machine $join WHERE lastseen > $online AND ($where)", $args);
-		$used = Database::queryFirst("SELECT Count(*) AS val FROM machine $join WHERE lastseen > $online AND logintime <> 0 AND ($where)", $args);
+		$on = Database::queryFirst("SELECT Count(*) AS val FROM machine $join WHERE lastboot <> 0 AND ($where)", $args);
+		$used = Database::queryFirst("SELECT Count(*) AS val FROM machine $join WHERE lastboot <> 0 AND logintime <> 0 AND ($where)", $args);
 		$hdd = Database::queryFirst("SELECT Count(*) AS val FROM machine $join WHERE badsectors >= 10 AND ($where)", $args);
 		if ($on['val'] != 0) {
 			$usedpercent = round($used['val'] / $on['val'] * 100);
@@ -500,7 +501,7 @@ class Page_Statistics extends Page
 		$rows = array();
 		$NOW = time();
 		while ($row = $res->fetch(PDO::FETCH_ASSOC)) {
-			if ($NOW - $row['lastseen'] > 610) {
+			if ($row['lastboot'] == 0) {
 				$row['state_off'] = true;
 			} elseif ($row['logintime'] == 0) {
 				$row['state_idle'] = true;
@@ -620,9 +621,14 @@ class Page_Statistics extends Page
 		$client = Database::queryFirst('SELECT machineuuid, locationid, macaddr, clientip, firstseen, lastseen, logintime, lastboot,'
 			. ' mbram, kvmstate, cpumodel, id44mb, data, hostname, notes FROM machine WHERE machineuuid = :uuid',
 			array('uuid' => $uuid));
+		// Hack: Get raw collected data
+		if (Request::get('raw', false)) {
+			Header('Content-Type: text/plain; charset=utf-8');
+			die($client['data']);
+		}
 		// Mangle fields
 		$NOW = time();
-		if ($NOW - $client['lastseen'] > 610) {
+		if ($client['lastboot'] == 0) {
 			$client['state_off'] = true;
 		} elseif ($client['logintime'] == 0) {
 			$client['state_idle'] = true;
@@ -632,10 +638,14 @@ class Page_Statistics extends Page
 		}
 		$client['firstseen_s'] = date('d.m.Y H:i', $client['firstseen']);
 		$client['lastseen_s'] = date('d.m.Y H:i', $client['lastseen']);
-		$uptime = $NOW - $client['lastboot'];
-		$client['lastboot_s'] = date('d.m.Y H:i', $client['lastboot']);
-		if (!isset($client['state_off']) || !$client['state_off']) {
-			$client['lastboot_s'] .= ' (Up ' . floor($uptime / 86400) . 'd ' . gmdate('H:i', $uptime) . ')';
+		if ($client['lastboot'] == 0) {
+			$client['lastboot_s'] = '-';
+		} else {
+			$uptime = $NOW - $client['lastboot'];
+			$client['lastboot_s'] = date('d.m.Y H:i', $client['lastboot']);
+			if (!isset($client['state_off']) || !$client['state_off']) {
+				$client['lastboot_s'] .= ' (Up ' . floor($uptime / 86400) . 'd ' . gmdate('H:i', $uptime) . ')';
+			}
 		}
 		$client['logintime_s'] = date('d.m.Y H:i', $client['logintime']);
 		$client['gbram'] = round(round($client['mbram'] / 500) / 2, 1);
-- 
cgit v1.2.3-55-g7522


From bf9db9aef8966de85652393cbcc28053dd2ffed3 Mon Sep 17 00:00:00 2001
From: Simon Rettberg
Date: Mon, 22 Aug 2016 12:13:19 +0200
Subject: [dozmod] Show wasted space in blockstats

---
 modules-available/dozmod/page.inc.php                      | 3 +++
 modules-available/dozmod/templates/blockstats-details.html | 2 +-
 modules-available/dozmod/templates/blockstats.html         | 6 +++++-
 3 files changed, 9 insertions(+), 2 deletions(-)

(limited to 'modules-available')

diff --git a/modules-available/dozmod/page.inc.php b/modules-available/dozmod/page.inc.php
index dfb49e0b..6c0c8a6b 100644
--- a/modules-available/dozmod/page.inc.php
+++ b/modules-available/dozmod/page.inc.php
@@ -126,11 +126,14 @@ class Page_DozMod extends Page
 		$res = Database::simpleQuery("SELECT blocksha1, blocksize, Count(*) AS blockcount FROM sat.imageblock"
 			. " GROUP BY blocksha1, blocksize HAVING blockcount > 1 ORDER BY blockcount DESC, blocksha1 ASC");
 		$data = array('hashes' => array());
+		$spaceWasted = 0;
 		while ($row = $res->fetch(PDO::FETCH_ASSOC)) {
 			$row['hash_hex'] = bin2hex($row['blocksha1']);
 			$row['blocksize_s'] = Util::readableFileSize($row['blocksize']);
 			$data['hashes'][] = $row;
+			$spaceWasted += $row['blocksize'] * ($row['blockcount'] - 1);
 		}
+		$data['spacewasted'] = Util::readableFileSize($spaceWasted);
 		Render::addTemplate('blockstats', $data);
 	}
 
diff --git a/modules-available/dozmod/templates/blockstats-details.html b/modules-available/dozmod/templates/blockstats-details.html
index 68e03fbc..e9466d94 100644
--- a/modules-available/dozmod/templates/blockstats-details.html
+++ b/modules-available/dozmod/templates/blockstats-details.html
@@ -1,6 +1,6 @@
 <table class="table table-bordered table-striped">
 	<tr>
-		<th>{{lang_imageName}}</th>
+		<th>{{lang_image}}</th>
 		<th>{{lang_createTime}}</th>
 		<th>{{lang_fileSize}}</th>
 		<th>{{lang_blockCount}}</th>
diff --git a/modules-available/dozmod/templates/blockstats.html b/modules-available/dozmod/templates/blockstats.html
index 82a66146..cba3b476 100644
--- a/modules-available/dozmod/templates/blockstats.html
+++ b/modules-available/dozmod/templates/blockstats.html
@@ -1,8 +1,12 @@
+<div class="alert alert-info">
+	{{lang_spaceWastedDuplication}}: {{spacewasted}}
+</div>
+
 <table class="table table-condensed table-striped" style="width: auto">
 	<tr>
 		<th>{{lang_hash}}</th>
 		<th>{{lang_size}}</th>
-		<th>{{lang_count}}</th>
+		<th>{{lang_blockCount}}</th>
 	</tr>
 	{{#hashes}}
 	<tr>
-- 
cgit v1.2.3-55-g7522


From b766e81d347c829ac13f3aef6558f268d93d5d8d Mon Sep 17 00:00:00 2001
From: Simon Rettberg
Date: Mon, 22 Aug 2016 12:13:28 +0200
Subject: [dozmod] Update translations

---
 modules-available/dozmod/lang/de/template-tags.json | 4 ++++
 modules-available/dozmod/lang/en/template-tags.json | 4 ++++
 2 files changed, 8 insertions(+)

(limited to 'modules-available')

diff --git a/modules-available/dozmod/lang/de/template-tags.json b/modules-available/dozmod/lang/de/template-tags.json
index fe558ed5..92977639 100644
--- a/modules-available/dozmod/lang/de/template-tags.json
+++ b/modules-available/dozmod/lang/de/template-tags.json
@@ -1,6 +1,7 @@
 {
     "lang_actionTarget": "Aktionsziel",
     "lang_asteriskRequired": "Felder mit (*) sind erforderlich",
+    "lang_blockCount": "Anzahl Bl\u00f6cke",
     "lang_canLogin": "Nutzer dieser Einrichtung k\u00f6nnen sich am Satelliten anmelden",
     "lang_createTime": "Erstellt",
     "lang_currentFilter": "Aktueller Filter",
@@ -21,8 +22,10 @@
     "lang_emailNotifications": "EMail-Benachrichtigungen aktiviert",
     "lang_error": "Fehler",
     "lang_event": "Ereignis",
+    "lang_fileSize": "Dateigr\u00f6\u00dfe",
     "lang_followingPlaceholdersUnused": "Folgende Platzhalter m\u00fcssen im Template verwendet werden",
     "lang_hasNewer": "Neuere Version existiert",
+    "lang_hash": "Hash",
     "lang_heading": "Zu l\u00f6schende VM-Versionen",
     "lang_host": "Host",
     "lang_image": "VM",
@@ -56,6 +59,7 @@
     "lang_senderAddress": "Absenderadresse",
     "lang_senderName": "Absender Anzeigename",
     "lang_size": "Gr\u00f6\u00dfe",
+    "lang_spaceWastedDuplication": "Potentiell durch mehrfach vorkommende Bl\u00f6cke belegter Speicherplatz",
     "lang_ssl": "SSL-Modus",
     "lang_sslExplicit": "Explizites SSL (\"STARTTLS\")",
     "lang_sslImplicit": "Implizites SSL",
diff --git a/modules-available/dozmod/lang/en/template-tags.json b/modules-available/dozmod/lang/en/template-tags.json
index 6a0434a3..396cb839 100644
--- a/modules-available/dozmod/lang/en/template-tags.json
+++ b/modules-available/dozmod/lang/en/template-tags.json
@@ -1,6 +1,7 @@
 {
     "lang_actionTarget": "Action target",
     "lang_asteriskRequired": "Fields marked with (*) are required",
+    "lang_blockCount": "Block count",
     "lang_canLogin": "Members of this organization can login",
     "lang_createTime": "Created",
     "lang_currentFilter": "Current filter",
@@ -21,8 +22,10 @@
     "lang_emailNotifications": "E-Mail notifications enabled",
     "lang_error": "Error",
     "lang_event": "Event",
+    "lang_fileSize": "File size",
     "lang_followingPlaceholdersUnused": "The following placeholders are not being used",
     "lang_hasNewer": "Newer version exists",
+    "lang_hash": "Hash",
     "lang_heading": "Images marked for deletion",
     "lang_host": "Host",
     "lang_image": "VM",
@@ -56,6 +59,7 @@
     "lang_senderAddress": "Sender address",
     "lang_senderName": "Sender's display name",
     "lang_size": "Size",
+    "lang_spaceWastedDuplication": "Potentially wasted space by duplicate blocks",
     "lang_ssl": "SSL mode",
     "lang_sslExplicit": "Explicit SSL (\"STARTTLS\")",
     "lang_sslImplicit": "Implicit SSL",
-- 
cgit v1.2.3-55-g7522