From e9aa7ceb89c487f5d2c2bb102f67618ebfcae08b Mon Sep 17 00:00:00 2001 From: Simon Rettberg Date: Mon, 24 Apr 2017 16:05:13 +0200 Subject: [statistics_reporting] Extend remote report by hwstats, date ranges, userstats This implements #3108 --- .../statistics_reporting/hooks/cron.inc.php | 3 +- .../statistics_reporting/inc/getdata.inc.php | 1 + .../statistics_reporting/inc/queries.inc.php | 63 ++++++++++++++++++++++ .../statistics_reporting/inc/remotereport.inc.php | 62 +++++++++++++++++---- .../statistics_reporting/page.inc.php | 2 +- 5 files changed, 119 insertions(+), 12 deletions(-) (limited to 'modules-available/statistics_reporting') diff --git a/modules-available/statistics_reporting/hooks/cron.inc.php b/modules-available/statistics_reporting/hooks/cron.inc.php index afb18a23..14597f7d 100644 --- a/modules-available/statistics_reporting/hooks/cron.inc.php +++ b/modules-available/statistics_reporting/hooks/cron.inc.php @@ -7,10 +7,9 @@ if (RemoteReport::isReportingEnabled()) { while ($nextReporting <= time()) { RemoteReport::writeNextReportingTimestamp(); - $from = strtotime("-7 days", $nextReporting); $to = $nextReporting; - $statisticsReport = json_encode(RemoteReport::generateReport($from, $to)); + $statisticsReport = json_encode(RemoteReport::generateReport($to)); $params = array("action" => "statistics", "data" => $statisticsReport); diff --git a/modules-available/statistics_reporting/inc/getdata.inc.php b/modules-available/statistics_reporting/inc/getdata.inc.php index da3a9a26..ae4d5aa4 100644 --- a/modules-available/statistics_reporting/inc/getdata.inc.php +++ b/modules-available/statistics_reporting/inc/getdata.inc.php @@ -29,6 +29,7 @@ class GetData $data["medianSessionLength_s"] = self::formatSeconds($data["medianSessionLength"]); $data["totalOffTime_s"] = self::formatSeconds($data["totalOffTime"]); } + $data['uniqueUsers'] = Queries::getUniqueUserCount(self::$from, self::$to); return $data; } diff --git a/modules-available/statistics_reporting/inc/queries.inc.php b/modules-available/statistics_reporting/inc/queries.inc.php index 2269e764..1a125c9e 100644 --- a/modules-available/statistics_reporting/inc/queries.inc.php +++ b/modules-available/statistics_reporting/inc/queries.inc.php @@ -213,5 +213,68 @@ class Queries "; return "(".$queryString.")"; } + + public static function getDozmodStats($from, $to) + { + if (!Module::isAvailable('dozmod')) + return array('disabled' => true); + + $return = array(); + $return['vms'] = Database::queryFirst("SELECT Count(*) AS `total`, Sum(If(createtime >= $from, 1, 0)) AS `new`, + Sum(If(updatetime >= $from, 1, 0)) AS `updated`, Sum(If(latestversionid IS NOT NULL, 1, 0)) AS `valid` + FROM sat.imagebase + WHERE createtime <= $to"); + $return['lectures'] = Database::queryFirst("SELECT Count(*) AS `total`, Sum(If(createtime >= $from, 1, 0)) AS `new`, + Sum(If(updatetime >= $from, 1, 0)) AS `updated`, + Sum(If((($from BETWEEN starttime AND endtime) OR ($to BETWEEN starttime AND endtime)) AND isenabled <> 0, 1, 0)) AS `valid` + FROM sat.lecture + WHERE createtime <= $to"); + $return['users'] = Database::queryFirst("SELECT Count(*) AS `total`, Count(DISTINCT organizationid) AS `organizations` + FROM sat.user + WHERE lastlogin >= $from"); + return $return; + } + + public static function getAggregatedMachineStats($from) + { + $return = array(); + $return['location'] = Database::queryAll("SELECT MD5(CONCAT(locationid, :salt)) AS `location`, Count(*) AS `count` + FROM machine + WHERE lastseen >= $from + GROUP BY locationid", + array('salt' => GetData::$salt)); + $prev = 0; + $str = ' '; + foreach (array(0.5, 1, 1.5, 2, 3, 4, 6, 8, 10, 12, 16, 20, 24, 28, 32, 40, 48, 64, 72, 80, 88, 96, 128, 192, 256) as $val) { + $str .= 'WHEN mbram < ' . round(($val + $prev) * 512) . " THEN '" . $prev . "' "; + $prev = $val; + } + $return['ram'] = Database::queryAll("SELECT CASE $str ELSE 'OVER 9000' END AS `gbram`, Count(*) AS `total` + FROM machine + WHERE lastseen >= $from + GROUP BY gbram"); + foreach (array('cpumodel', 'systemmodel', 'realcores', 'kvmstate') as $key) { + $return[$key] = Database::queryAll("SELECT $key, Count(*) AS `total` + FROM machine + WHERE lastseen >= $from + GROUP BY $key"); + } + return $return; + } + + /** + * @param int $from start timestamp + * @param int $to end timestamp + * @return int count of user active in timespan + */ + public static function getUniqueUserCount($from, $to) + { + $res = Database::queryFirst("SELECT Count(DISTINCT username) as `total` + FROM statistic + WHERE (dateline BETWEEN $from AND $to) AND typeid = '.vmchooser-session-name' + GROUP BY username"); + return (int)$res['total']; + } + } diff --git a/modules-available/statistics_reporting/inc/remotereport.inc.php b/modules-available/statistics_reporting/inc/remotereport.inc.php index 4c5f604b..68a3e95e 100644 --- a/modules-available/statistics_reporting/inc/remotereport.inc.php +++ b/modules-available/statistics_reporting/inc/remotereport.inc.php @@ -63,19 +63,63 @@ class RemoteReport * Generate the multi-dimensional array containing the anonymized * (weekly) statistics to report. * - * @param int $from start timestamp * @param int $to end timestamp + * @param int[] $days list of days to generate aggregated stats for * @return array wrapped up statistics, ready for reporting */ - public static function generateReport($from, $to) { - GetData::$from = $from; - GetData::$to = $to; + public static function generateReport($to, $days = false) { + if ($days === false) { + $days = [7, 30, 90]; + } GetData::$salt = bin2hex(Util::randomBytes(20, false)); - $data = GetData::total(GETDATA_ANONYMOUS); - $data['perLocation'] = GetData::perLocation(GETDATA_ANONYMOUS); - $data['perVM'] = GetData::perVM(GETDATA_ANONYMOUS); - $data['tsFrom'] = $from; - $data['tsTo'] = $to; + GetData::$lowerTimeBound = 7; + GetData::$upperTimeBound = 20; + $result = array(); + foreach ($days as $day) { + if (isset($result['days' . $day])) + continue; + $from = strtotime("-{$day} days", $to); + GetData::$from = $from; + GetData::$to = $to; + $data = array('total' => GetData::total(GETDATA_ANONYMOUS)); + $data['perLocation'] = GetData::perLocation(GETDATA_ANONYMOUS); + $data['perVM'] = GetData::perVM(GETDATA_ANONYMOUS); + $data['tsFrom'] = $from; + $data['tsTo'] = $to; + $data['dozmod'] = Queries::getDozmodStats($from, $to); + $data['machines'] = Queries::getAggregatedMachineStats($from); + $result['days' . $day] = $data; + } + $result['server'] = self::getLocalHardware(); + $result['version'] = CONFIG_FOOTER; + return $result; + } + + private function getLocalHardware() + { + $cpuInfo = file_get_contents('/proc/cpuinfo'); + $uptime = file_get_contents('/proc/uptime'); + $memInfo = file_get_contents('/proc/meminfo'); + preg_match_all('/\b(\w+):\s+(\d+)\s/s', $memInfo, $out, PREG_SET_ORDER); + $mem = array(); + foreach ($out as $e) { + $mem[$e[1]] = $e[2]; + } + // + $data = array(); + $data['cpuCount'] = preg_match_all('/\bprocessor\s+:\s+(.*)$/m', $cpuInfo, $out); + if ($data['cpuCount'] > 0) { + $data['cpuModel'] = $out[1][0]; + } + if (preg_match('/^(\d+)\D/', $uptime, $out)) { + $data['uptime'] = $out[1]; + } + if (isset($mem['MemTotal']) && isset($mem['MemFree']) && isset($mem['SwapTotal'])) { + $data['memTotal'] = $mem['MemTotal']; + $data['memFree'] = ($mem['MemFree'] + $mem['Buffers'] + $mem['Cached']); + $data['swapTotal'] = $mem['SwapTotal']; + $data['swapUsed'] = ($mem['SwapTotal'] - $mem['SwapFree']); + } return $data; } diff --git a/modules-available/statistics_reporting/page.inc.php b/modules-available/statistics_reporting/page.inc.php index 6bd908a5..52accaea 100644 --- a/modules-available/statistics_reporting/page.inc.php +++ b/modules-available/statistics_reporting/page.inc.php @@ -58,7 +58,7 @@ class Page_Statistics_Reporting extends Page // Get report - fetch data exactly the way it would automatically be reported // so the user can know what is going on if ($this->action === 'getreport') { - $report = RemoteReport::generateReport(strtotime('-7 days'), time('now')); + $report = RemoteReport::generateReport(time()); Header('Content-Disposition: attachment; filename=remote-report.json'); Header('Content-Type: application/json; charset=utf-8'); die(json_encode($report)); -- cgit v1.2.3-55-g7522 From 534e8cc91072acb55a847f093d37e3eccbb364cc Mon Sep 17 00:00:00 2001 From: Simon Rettberg Date: Fri, 28 Apr 2017 10:24:36 +0200 Subject: Move rebootcontrol, locationinfo to new beta section, statistics_reporting to status Closes #3122 --- modules-available/locationinfo/config.json | 2 +- modules-available/main/category-icons.json | 3 ++- modules-available/main/lang/de/categories.json | 1 + modules-available/main/lang/en/categories.json | 1 + modules-available/rebootcontrol/config.json | 2 +- modules-available/statistics_reporting/config.json | 2 +- 6 files changed, 7 insertions(+), 4 deletions(-) (limited to 'modules-available/statistics_reporting') diff --git a/modules-available/locationinfo/config.json b/modules-available/locationinfo/config.json index d4cab3f0..87825809 100644 --- a/modules-available/locationinfo/config.json +++ b/modules-available/locationinfo/config.json @@ -1,4 +1,4 @@ { - "category":"main.content", + "category":"main.beta", "dependencies": ["js_jqueryui", "bootstrap_timepicker", "locations", "bootstrap_switch"] } diff --git a/modules-available/main/category-icons.json b/modules-available/main/category-icons.json index 08cb2a64..f790089e 100644 --- a/modules-available/main/category-icons.json +++ b/modules-available/main/category-icons.json @@ -4,5 +4,6 @@ "settings-client":"cog", "settings-server":"cog", "status":"tasks", - "users":"user" + "users":"user", + "beta":"flash" } diff --git a/modules-available/main/lang/de/categories.json b/modules-available/main/lang/de/categories.json index 8fd292ec..71f149ec 100644 --- a/modules-available/main/lang/de/categories.json +++ b/modules-available/main/lang/de/categories.json @@ -1,4 +1,5 @@ { + "beta": "Beta", "content": "Inhalt", "settings-client": "Einstellungen (Client)", "settings-server": "Einstellungen (Server)", diff --git a/modules-available/main/lang/en/categories.json b/modules-available/main/lang/en/categories.json index defbd652..9dfa0404 100644 --- a/modules-available/main/lang/en/categories.json +++ b/modules-available/main/lang/en/categories.json @@ -1,4 +1,5 @@ { + "beta": "Beta", "content": "Content", "settings-client": "Settings (Client)", "settings-server": "Settings (Server)", diff --git a/modules-available/rebootcontrol/config.json b/modules-available/rebootcontrol/config.json index 2cc05822..d8ab5868 100644 --- a/modules-available/rebootcontrol/config.json +++ b/modules-available/rebootcontrol/config.json @@ -1,4 +1,4 @@ { - "category":"main.content", + "category":"main.beta", "dependencies": [ "locations", "js_stupidtable" ] } diff --git a/modules-available/statistics_reporting/config.json b/modules-available/statistics_reporting/config.json index f9627cdb..78ca35ba 100644 --- a/modules-available/statistics_reporting/config.json +++ b/modules-available/statistics_reporting/config.json @@ -1,4 +1,4 @@ { - "category": "main.content", + "category": "main.status", "dependencies": [ "statistics", "locations", "js_stupidtable", "js_jqueryui" ] } -- cgit v1.2.3-55-g7522 From 850c510472756947295d03f17fb8f27884caaa74 Mon Sep 17 00:00:00 2001 From: Simon Rettberg Date: Tue, 2 May 2017 15:31:35 +0200 Subject: [statistics_reporting] Reporting is opt-out now --- .../statistics_reporting/inc/remotereport.inc.php | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) (limited to 'modules-available/statistics_reporting') diff --git a/modules-available/statistics_reporting/inc/remotereport.inc.php b/modules-available/statistics_reporting/inc/remotereport.inc.php index 68a3e95e..e94e9b93 100644 --- a/modules-available/statistics_reporting/inc/remotereport.inc.php +++ b/modules-available/statistics_reporting/inc/remotereport.inc.php @@ -13,18 +13,19 @@ class RemoteReport */ public static function setReportingEnabled($isEnabled) { - $value = ($isEnabled === true || $isEnabled === 'on') ? 'on' : ''; - Property::set(self::ENABLED_ID, $value, 60 * 24 * 14); + $value = ($isEnabled === true || $isEnabled === 'on') ? 'on' : 'off'; + Property::set(self::ENABLED_ID, $value); } /** * Returns whether remote reporting is enabled or not. + * Defaults to on. * * @return bool true if reporting is on, false if off */ public static function isReportingEnabled() { - return Property::get(self::ENABLED_ID, false) === 'on'; + return Property::get(self::ENABLED_ID, 'on') === 'on'; } /** @@ -42,9 +43,9 @@ class RemoteReport // schedule for next time self::writeNextReportingTimestamp(); $ts = Property::get(self::NEXT_SUBMIT_ID, 0); - } elseif ($ts < strtotime('last monday')) { - // Too long ago, move forward to last monday - $ts = strtotime('last monday'); + } elseif ($ts < strtotime('last monday - 14 days')) { + // Too long ago, move forward + $ts = strtotime('last monday - 14 days'); } return $ts; } -- cgit v1.2.3-55-g7522 From 6ed17bee375191ebfddad14379eea2efea5891fa Mon Sep 17 00:00:00 2001 From: Simon Rettberg Date: Fri, 5 May 2017 13:04:56 +0200 Subject: [statistics_reporting] Add link to wiki explaining remote reporting in detail --- modules-available/statistics_reporting/lang/de/template-tags.json | 3 ++- modules-available/statistics_reporting/lang/en/template-tags.json | 1 + modules-available/statistics_reporting/templates/columnChooser.html | 1 + 3 files changed, 4 insertions(+), 1 deletion(-) (limited to 'modules-available/statistics_reporting') diff --git a/modules-available/statistics_reporting/lang/de/template-tags.json b/modules-available/statistics_reporting/lang/de/template-tags.json index 3801f3a6..fb624fad 100644 --- a/modules-available/statistics_reporting/lang/de/template-tags.json +++ b/modules-available/statistics_reporting/lang/de/template-tags.json @@ -10,7 +10,8 @@ "lang_location": "Ort", "lang_longSessions": "Sitzungen \u2265 60s", "lang_medianSessionLength": "Sitzungsdauer Median", - "lang_reportingDescription": "Helfen Sie uns bwLehrpool durch das w\u00f6chentliche Verschicken eines anonymisierten Statistikberichts zu verbessern. Wenn Sie den Inhalt eines solchen Reports genauer inspizieren wollen, k\u00f6nnen Sie \u00fcber den untenstehenden Button einen aktuellen Report ihres Servers herunterladen.", + "lang_reportMoreInfo": "Weitere Informationen...", + "lang_reportingDescription": "Helfen Sie uns bwLehrpool durch das w\u00f6chentliche Verschicken eines anonymisierten Statistikberichts zu verbessern. Wenn Sie den Inhalt eines solchen Reports genauer inspizieren wollen, k\u00f6nnen Sie \u00fcber den untenstehenden Button einen aktuellen Report Ihres Servers herunterladen.", "lang_reportingLabel": "Anonymisierte Nutzungsstatistiken \u00fcbermitteln", "lang_sessions": "Sitzungen", "lang_shortSessions": "Sitzungen < 60s", diff --git a/modules-available/statistics_reporting/lang/en/template-tags.json b/modules-available/statistics_reporting/lang/en/template-tags.json index a4865931..4e33ffed 100644 --- a/modules-available/statistics_reporting/lang/en/template-tags.json +++ b/modules-available/statistics_reporting/lang/en/template-tags.json @@ -10,6 +10,7 @@ "lang_location": "Location", "lang_longSessions": "Sessions \u2265 60s", "lang_medianSessionLength": "Median Session Length", + "lang_reportMoreInfo": "More information...", "lang_reportingDescription": "Help us improve bwLehrpool by automatically sending an anonymized statistics report once a week. If you want to check what data the report contains, you can download such a report for reference below.", "lang_reportingLabel": "Send anonymized usage statistics", "lang_sessions": "Sessions", diff --git a/modules-available/statistics_reporting/templates/columnChooser.html b/modules-available/statistics_reporting/templates/columnChooser.html index a5ac828b..e4069be9 100644 --- a/modules-available/statistics_reporting/templates/columnChooser.html +++ b/modules-available/statistics_reporting/templates/columnChooser.html @@ -76,6 +76,7 @@

{{lang_reportingDescription}}

+

{{lang_reportMoreInfo}}

{{lang_downloadReport}}
-- cgit v1.2.3-55-g7522