[dnbd3/vmstore] Add first version of speedtest/benchmark GUI

6 files changed, 423 insertions, 1 deletions

diff --git a/modules-available/vmstore/inc/vmstorebenchmark.inc.php b/modules-available/vmstore/inc/vmstorebenchmark.inc.php
new file mode 100644
index 00000000..30b0c65c
--- /dev/null
+++ b/modules-available/vmstore/inc/vmstorebenchmark.inc.php
@@ -0,0 +1,65 @@
+<?php
+
+class VmStoreBenchmark
+{
+
+	/**
+	 * @param string[] $machineUuids List of UUIDs
+	 * @return void
+	 */
+	public static function prepareSelectDialog(array $uuids)
+	{
+		Module::isAvailable('rebootcontrol');
+		User::assertPermission('.vmstore.benchmark');
+		$uuids = array_values(Request::post('uuid', Request::REQUIRED, 'array'));
+		$machines = RebootUtils::getFilteredMachineList($uuids, '.vmstore.benchmark');
+		if ($machines === false)
+			return;
+		$machines = array_column($machines, 'machineuuid');
+		$id = mt_rand() . time();
+		Session::set('benchmark-' . $id, ['machines' => $machines], 60);
+		Util::redirect('?do=vmstore&show=benchmark&action=select&id=' . $id);
+	}
+
+	/**
+	 * @param array $machineUuids
+	 * @param string $image
+	 * @param bool $nfs
+	 * @param int $start timestamp when the clients should start
+	 * @return ?string taskId, or null on error
+	 */
+	public static function start(string $id, array $machineUuids, string $image, bool $nfs, int &$start)
+	{
+		Module::isAvailable('rebootcontrol');
+		$clients = Database::queryAll('SELECT machineuuid, clientip FROM machine WHERE machineuuid IN (:uuids)',
+			['uuids' => $machineUuids]);
+		if (empty($clients)) {
+			ErrorHandler::traceError('Cannot start benchmark: No matching clients');
+		}
+		// The more clients we have, the longer it takes to SSH into all of them.
+		// As of 2022, RemoteExec processes 4 clients in parallel
+		$start = ceil(count($clients) / 4 + 5 + time());
+		$nfsOpt = $nfs ? '--nfs' : '';
+		$command = <<<COMMAND
+(
+exec &> /dev/null < /dev/null
+setsid
+image_speedcheck --start $start --console $nfsOpt --file "$image" > "/tmp/speedcheck-$id"
+) &
+COMMAND;
+		$task = RebootControl::runScript($clients, $command);
+		return $task['id'] ?? null;
+	}
+
+	public static function parseBenchLine(string $line): array
+	{
+		$out = ['cpu' => [], 'net' => []];
+		foreach (explode(',', $line) as $elem) {
+			$elem = explode('+', $elem);
+			$out['net'][] = ['x' => (int)$elem[0], 'y' => (int)$elem[1]];
+			//$out['cpu'][] = ['x' => $elem[0], 'y' => $elem[2]];
+		}
+		return $out;
+	}
+
+}
\ No newline at end of file
diff --git a/modules-available/vmstore/page.inc.php b/modules-available/vmstore/page.inc.php
index 60a9d60a..41e7e990 100644
--- a/modules-available/vmstore/page.inc.php
+++ b/modules-available/vmstore/page.inc.php
@@ -11,6 +11,19 @@ class Page_VmStore extends Page
 	{
 		User::load();
 
+		if (User::hasPermission('edit')) {
+			Dashboard::addSubmenu('?do=vmstore', Dictionary::translate('menu_edit', true));
+		}
+		if (User::hasPermission('benchmark')) {
+			Dashboard::addSubmenu('?do=vmstore&show=benchmark', Dictionary::translate('menu_benchmark', true));
+		}
+
+		if (Request::any('show') === 'benchmark') {
+			User::assertPermission('benchmark');
+			$this->benchmarkDoPreprocess();
+			return;
+		}
+
 		User::assertPermission('edit');
 		
 		$action = Request::post('action');
@@ -22,6 +35,11 @@ class Page_VmStore extends Page
 	
 	protected function doRender()
 	{
+		if (Request::any('show') === 'benchmark') {
+			$this->benchmarkDoRender();
+			return;
+		}
+
 		$action = Request::post('action');
 		if ($action === 'setstore' && !Taskmanager::isFailed(Taskmanager::status($this->mountTask))) {
 			Render::addTemplate('mount', array(
@@ -68,4 +86,192 @@ class Page_VmStore extends Page
 		}
 	}
 
+	private function benchmarkDoPreprocess()
+	{
+		if (!Module::isAvailable('rebootcontrol')) {
+			ErrorHandler::traceError('rebootcontrol module not enabled');
+		}
+		if (Request::post('action') === 'start') {
+			$this->benchmarkActionStart();
+		}
+	}
+
+	private function benchmarkDoRender()
+	{
+		switch (Request::get('action')) {
+		case 'select':
+			$this->benchmarkShowImageSelect();
+			break;
+		case 'result':
+			$this->benchmarkShowResult();
+		}
+	}
+
+	private function benchmarkActionStart()
+	{
+		Module::isAvailable('dnbd3');
+		$id = Request::post('id', Request::REQUIRED, 'string');
+		$data = Session::get('benchmark-' . $id);
+		if (!isset($data['machines'])) {
+			Message::addError('invalid-benchmark-job', $id);
+			return;
+		}
+		if (isset($data['task'])) {
+			if ($data['task'] === 'inprogress') {
+				// Let's hope the proper ID gets written in a short while
+				sleep(1);
+			}
+			Util::redirect('?do=vmstore&show=benchmark&action=result&id=' . $id);
+		}
+		$nfs = !Dnbd3::isEnabled();
+		$data['image'] = Request::post('image', Request::REQUIRED, 'string');
+		// Save once first to minimize race window
+		$data['task'] = 'inprogress';
+		Session::set('benchmark-' . $id, $data, 60);
+		Session::saveExtraData();
+		$start = 0;
+		$data['task'] = VmStoreBenchmark::start($id, $data['machines'], $data['image'], $nfs, $start);
+		if ($data['task'] === null) {
+			$data['task'] = 'failed';
+		} else {
+			// Test is 2x 30 seconds
+			$data['expected'] = $start + 60;
+		}
+		error_log('Saving: ' . json_encode($data));
+		Session::set('benchmark-' . $id, $data, 60);
+		Util::redirect('?do=vmstore&show=benchmark&action=result&id=' . $id);
+	}
+
+	private function benchmarkShowImageSelect()
+	{
+		$id = Request::get('id', Request::REQUIRED, 'string');
+		$data = Session::get('benchmark-' . $id);
+		if (!isset($data['machines'])) {
+			Message::addError('invalid-benchmark-job', $id);
+			return;
+		}
+		if (isset($data['task'])) {
+			Message::addWarning('benchmark-already-started');
+			Util::redirect('?do=vmstore&show=benchmark&action=result&id=' . $id);
+		}
+		Module::isAvailable('dnbd3');
+		$lookup = Dnbd3::getActiveServers();
+		$list = Dnbd3Rpc::getStatsMulti(array_keys($lookup), [Dnbd3Rpc::QUERY_IMAGES]);
+		if (empty($list)) {
+			Message::addError('dnbd3-failed');
+			Util::redirect('?do=vmstore');
+		}
+		$images = [];
+		foreach ($list as $json) {
+			foreach ($json['images'] as $img) {
+				$name = $img['name'] . ':' . $img['rid'];
+				if (!isset($images[$name])) {
+					$images[$name] = [
+						'users' => 0,
+						'size' => Util::readableFileSize($img['size'], 1),
+						'name' => $name,
+						'id' => count($images)
+					];
+				}
+				$images[$name]['users'] += $img['users'];
+			}
+		}
+		ArrayUtil::sortByColumn($images, 'users', SORT_NUMERIC | SORT_DESC);
+		Render::addTemplate('benchmark-imgselect', ['id' => $id, 'list' => array_values($images)]);
+	}
+
+	private function benchmarkShowResult()
+	{
+		$id = Request::get('id', Request::REQUIRED, 'string');
+		$data = Session::get('benchmark-' . $id);
+		if (!isset($data['machines'])) {
+			Message::addError('invalid-benchmark-job', $id);
+			return;
+		}
+		if (!isset($data['task'])) {
+			Message::addWarning('select-image-first');
+			Util::redirect('?do=vmstore&show=benchmark&action=select&id=' . $id);
+		}
+		if ($data['task'] === 'failed') {
+			Message::addError('benchmark-failed');
+			return;
+		}
+		$remaining = 0;
+		if ($data['task'] !== 'done') {
+			$remaining = $data['expected'] - time();
+			if ($remaining < 0) {
+				$remaining = 0;
+			}
+			$this->processRunningBenchmark($id, $data, $remaining === 0);
+			$refresh = $remaining;
+			Util::clamp($refresh, 2, 64);
+		}
+		$args = [
+			'id' => $id,
+			'result' => json_encode($data['result'] ?? []),
+			'wanted' => json_encode($data['machines']),
+		];
+		if ($remaining > 0) {
+			$args['remaining'] = $remaining;
+			$args['refresh'] = $refresh;
+		}
+		Module::isAvailable('js_chart');
+		Render::addTemplate('benchmark-result', $args);
+	}
+
+	private function processRunningBenchmark(string $id, array &$data, bool $timeout)
+	{
+		Module::isAvailable('rebootcontrol');
+		$changed = false;
+
+		$active = array_filter($data['machines'], function ($e) use ($data) { return !isset($data['result'][$e]); });
+		if (empty($active)) {
+			$timeout = true;
+		} else {
+			$command = <<<EOF
+grep -q '^Seq:' "/tmp/speedcheck-$id" && cat "/tmp/speedcheck-$id"
+EOF;
+			$task = RebootControl::runScript($active, $command);
+			$task = Taskmanager::waitComplete($task, 4000);
+			if ($task === false) {
+				$data['task'] = 'failed';
+				return;
+			}
+			if (!isset($data['result'])) {
+				$data['result'] = [];
+			}
+			$res =& $task['data'];
+			foreach ($res['result'] as $uuid => $out) {
+				if (isset($data['result'][$uuid]))
+					continue;
+				error_log(json_encode($out));
+				// Not finished, ignore
+				if (($out['state'] !== 'DONE' || $out['exitCode'] !== 0) && !$timeout)
+					continue;
+				$changed = true;
+				unset($client);
+				$client = ['machineuuid' => $uuid];
+				$data['result'][$uuid] =& $client;
+				if (preg_match_all("/^\+(\w{3}):(\d+),(.*)$/m", $out['stdout'], $modes, PREG_SET_ORDER)) {
+					foreach ($modes as $mode) {
+						$client[$mode[1]] = [
+							'start' => $mode[2],
+							'values' => VmStoreBenchmark::parseBenchLine($mode[3]),
+						];
+					}
+				}
+				$m = Database::queryFirst('SELECT clientip, hostname FROM machine WHERE machineuuid = :uuid',
+					['uuid' => $uuid]);
+				$client['name'] = empty($m['hostname']) ? $m['clientip'] : $m['hostname'];
+			}
+		}
+		if (count($data['result']) === count($data['machines']) || $timeout) {
+			$data['task'] = 'done';
+			$changed = true;
+		}
+		if ($changed) {
+			Session::set('benchmark-' . $id, $data);
+		}
+	}
+
 }
\ No newline at end of file
diff --git a/modules-available/vmstore/permissions/permissions.json b/modules-available/vmstore/permissions/permissions.json
index 8303fd02..0617c673 100644
--- a/modules-available/vmstore/permissions/permissions.json
+++ b/modules-available/vmstore/permissions/permissions.json
@@ -1,5 +1,8 @@
 {
     "edit": {
         "location-aware": false
+    },
+    "benchmark": {
+        "location-aware": true
     }
 }
\ No newline at end of file
diff --git a/modules-available/vmstore/templates/benchmark-imgselect.html b/modules-available/vmstore/templates/benchmark-imgselect.html
new file mode 100644
index 00000000..26ac898e
--- /dev/null
+++ b/modules-available/vmstore/templates/benchmark-imgselect.html
@@ -0,0 +1,41 @@
+<h1>{{lang_benchmark}}</h1>
+
+<h3>{{lang_selectImage}}</h3>
+
+<form role="form" method="post" action="?do=vmstore">
+	<input type="hidden" name="token" value="{{token}}">
+	<input type="hidden" name="show" value="benchmark">
+	<input type="hidden" name="id" value="{{id}}">
+
+	<table class="table">
+		<thead>
+		<tr>
+			<th>{{lang_image}}</th>
+			<th class="slx-smallcol">{{lang_users}}</th>
+			<th class="slx-smallcol">{{lang_size}}</th>
+		</tr>
+		</thead>
+		<tbody>
+		{{#list}}
+			<tr>
+				<td>
+					<div class="radio radio-inline">
+						<input type="radio" id="r-{{id}}" name="image" value="{{name}}">
+						<label for="r-{{id}}">{{name}}</label>
+					</div>
+				</td>
+				<td class="text-right">{{users}}</td>
+				<td class="text-right">{{size}}</td>
+			</tr>
+		{{/list}}
+		</tbody>
+	</table>
+
+	<div class="buttonbar text-right">
+		<button type="submit" name="action" value="start" class="btn btn-primary">
+			<span class="glyphicon glyphicon-play"></span>
+			{{lang_start}}
+		</button>
+	</div>
+
+</form>
\ No newline at end of file
diff --git a/modules-available/vmstore/templates/benchmark-result.html b/modules-available/vmstore/templates/benchmark-result.html
new file mode 100644
index 00000000..fc7f8a55
--- /dev/null
+++ b/modules-available/vmstore/templates/benchmark-result.html
@@ -0,0 +1,106 @@
+<h1>{{lang_benchmark}}</h1>
+
+<h2>{{lang_benchmarkResult}}</h2>
+
+{{#remaining}}
+<div class="alert alert-info">
+	{{lang_benchmarkSecondsReminaing}}: {{remaining}}
+</div>
+{{/remaining}}
+
+<div id="graphs"></div>
+
+<script>
+	{{#refresh}}
+		setTimeout(function() {
+			window.location.reload();
+		}, {{refresh}} * 1000);
+	{{/refresh}}
+
+	document.addEventListener('DOMContentLoaded', function() {
+		var result = {{{result}}};
+		var clients = {{{wanted}}};
+		var graphs = {};
+		function formatBytes(val) {
+			return Math.floor(val / 1024 / 1024) + "MB/s";
+		}
+		function renderX(val, index) {
+			return Math.floor(val / 1000) + 's';
+		}
+		function makeGraph(typeKey, resourceKey, caption) {
+			var uuid;
+			var ds = [];
+			var gmin = 0, lmax = 0;
+			var colors = [];
+			var cnt = 0;
+			for (uuid in result) {
+				if (gmin === 0 || result[uuid][typeKey].start < gmin) {
+					gmin = result[uuid][typeKey].start;
+				}
+				cnt++;
+			}
+			if (cnt === 1) {
+				colors.push('rgb(0, 128, 0)');
+			} else {
+				for (i = 0; i < cnt; ++i) {
+					colors.push('rgb(0, 128, ' + (i / (cnt - 1)) * 255 + ')');
+				}
+			}
+			var v, i, o, idx;
+			var sums = [];
+			for (uuid in result) {
+				o = result[uuid][typeKey].start - gmin; // Adjust according to earliest client
+				v = result[uuid][typeKey].values[resourceKey];
+				for (i = 0; i < v.length; ++i) {
+					v[i].x += o;
+					if (cnt > 1) {
+						idx = (v[i].x / 250) | 0;
+						if (sums[idx]) {
+							sums[idx] += v[i].y | 0;
+						} else {
+							sums[idx] = v[i].y | 0;
+						}
+					}
+				}
+				if (v[v.length-1].x > lmax) lmax = v[v.length-1].x; // Get max value
+				ds.push({data: v, label: result[uuid].name, borderColor: colors[ds.length], fill: false});
+			}
+			if (cnt > 1) {
+				ds.push({data: sums, label: 'Sum', borderColor: '#c00'});
+			}
+			if (!graphs[typeKey]) {
+				var $e = $('#graphs');
+				var $c = $('<canvas style="width:100%;height:250px">');
+				$e.append($('<h3>').text(caption));
+				$e.append($c);
+				var ls = [];
+				for (i = 0; i <= lmax; i += 250) ls.push(i); // Generate steps for graph
+				graphs[typeKey] = new Chart($c[0].getContext('2d'), {data: {datasets: ds, labels: ls}, type: 'scatter', options: {
+						animation: false,
+						responsive: true,
+						spanGaps: true,
+						borderWidth: 2,
+						pointBorderWidth: 0,
+						showLine: true,
+						scales: { y: { ticks: { callback: formatBytes }}, x: { ticks: { callback: renderX } } },
+						plugins: {
+							tooltip: { callbacks: { label: function(context) {
+										if (context.parsed.y !== null) {
+											return context.dataset.label + ": " + formatBytes(context.parsed.y);
+										}
+										return context.dataset.label;
+									}
+							}}}
+					}});
+			} else {
+				graphs[typeKey].data.datasets = ds;
+				graphs[typeKey].update();
+			}
+			console.log(graphs[typeKey].data);
+		}
+
+		makeGraph('SEQ', 'net', 'Sequential Reads');
+		makeGraph('RND', 'net', 'Random 1M');
+
+	});
+</script>
\ No newline at end of file
diff --git a/modules-available/vmstore/templates/page-vmstore.html b/modules-available/vmstore/templates/page-vmstore.html
index 0e1ad601..fa222631 100644
--- a/modules-available/vmstore/templates/page-vmstore.html
+++ b/modules-available/vmstore/templates/page-vmstore.html
@@ -1,10 +1,11 @@
+<h1>{{lang_vmLocation}}</h1>
+
 <form role="form" method="post" action="?do=VmStore">
 	<input type="text" name="prevent_autofill" id="prevent_autofill" value="" style="position:absolute;top:-2000px" tabindex="-1">
 	<input type="password" name="password_fake" id="password_fake" value=""   style="position:absolute;top:-2000px" tabindex="-1">
 	<input type="hidden" name="token" value="{{token}}">
 	<input type="hidden" name="action" value="setstore">
 
-	<h1>{{lang_vmLocation}}</h1>
 
 	<p>{{lang_vmLocationChoose}} <a class="btn btn-default" data-toggle="modal" data-target="#help-store"><span class="glyphicon glyphicon-question-sign"></span></a></p>