From cebc0c48fd86750eb9b45745a2d68c5e5d71d9f8 Mon Sep 17 00:00:00 2001 From: Simon Rettberg Date: Fri, 1 Jul 2022 17:51:36 +0200 Subject: [dnbd3/vmstore] Add first version of speedtest/benchmark GUI --- modules-available/dnbd3/inc/dnbd3.inc.php | 2 +- modules-available/dnbd3/page.inc.php | 2 +- .../vmstore/inc/vmstorebenchmark.inc.php | 65 +++++++ modules-available/vmstore/page.inc.php | 206 +++++++++++++++++++++ .../vmstore/permissions/permissions.json | 3 + .../vmstore/templates/benchmark-imgselect.html | 41 ++++ .../vmstore/templates/benchmark-result.html | 106 +++++++++++ .../vmstore/templates/page-vmstore.html | 3 +- 8 files changed, 425 insertions(+), 3 deletions(-) create mode 100644 modules-available/vmstore/inc/vmstorebenchmark.inc.php create mode 100644 modules-available/vmstore/templates/benchmark-imgselect.html create mode 100644 modules-available/vmstore/templates/benchmark-result.html diff --git a/modules-available/dnbd3/inc/dnbd3.inc.php b/modules-available/dnbd3/inc/dnbd3.inc.php index 0c640973..7f873f23 100644 --- a/modules-available/dnbd3/inc/dnbd3.inc.php +++ b/modules-available/dnbd3/inc/dnbd3.inc.php @@ -31,7 +31,7 @@ class Dnbd3 { $res = Database::simpleQuery('SELECT s.serverid, m.clientip, s.fixedip FROM dnbd3_server s LEFT JOIN machine m ON (s.machineuuid = m.machineuuid) - WHERE s.lastseen > :cutoff', ['cutoff' => time() - 310]); + WHERE s.lastseen > :cutoff', ['cutoff' => CONFIG_DEBUG ? 0 : time() - 310]); $lookup = []; foreach ($res as $row) { $lookup[$row['fixedip'] ?? $row['clientip'] ?? ''] = $row['serverid']; diff --git a/modules-available/dnbd3/page.inc.php b/modules-available/dnbd3/page.inc.php index 7173fc10..f8cabca5 100644 --- a/modules-available/dnbd3/page.inc.php +++ b/modules-available/dnbd3/page.inc.php @@ -623,7 +623,7 @@ class Page_Dnbd3 extends Page private function ajaxStats() { $lookup = Dnbd3::getActiveServers(); - $result = Dnbd3Rpc::getStatsMulti(array_keys($lookup)); + $result = Dnbd3Rpc::getStatsMulti(array_keys($lookup), [Dnbd3Rpc::QUERY_STATS]); $return = []; foreach ($result as $ip => $data) { $return[$lookup[$ip]] = $data; diff --git a/modules-available/vmstore/inc/vmstorebenchmark.inc.php b/modules-available/vmstore/inc/vmstorebenchmark.inc.php new file mode 100644 index 00000000..30b0c65c --- /dev/null +++ b/modules-available/vmstore/inc/vmstorebenchmark.inc.php @@ -0,0 +1,65 @@ + $machines], 60); + Util::redirect('?do=vmstore&show=benchmark&action=select&id=' . $id); + } + + /** + * @param array $machineUuids + * @param string $image + * @param bool $nfs + * @param int $start timestamp when the clients should start + * @return ?string taskId, or null on error + */ + public static function start(string $id, array $machineUuids, string $image, bool $nfs, int &$start) + { + Module::isAvailable('rebootcontrol'); + $clients = Database::queryAll('SELECT machineuuid, clientip FROM machine WHERE machineuuid IN (:uuids)', + ['uuids' => $machineUuids]); + if (empty($clients)) { + ErrorHandler::traceError('Cannot start benchmark: No matching clients'); + } + // The more clients we have, the longer it takes to SSH into all of them. + // As of 2022, RemoteExec processes 4 clients in parallel + $start = ceil(count($clients) / 4 + 5 + time()); + $nfsOpt = $nfs ? '--nfs' : ''; + $command = << /dev/null < /dev/null +setsid +image_speedcheck --start $start --console $nfsOpt --file "$image" > "/tmp/speedcheck-$id" +) & +COMMAND; + $task = RebootControl::runScript($clients, $command); + return $task['id'] ?? null; + } + + public static function parseBenchLine(string $line): array + { + $out = ['cpu' => [], 'net' => []]; + foreach (explode(',', $line) as $elem) { + $elem = explode('+', $elem); + $out['net'][] = ['x' => (int)$elem[0], 'y' => (int)$elem[1]]; + //$out['cpu'][] = ['x' => $elem[0], 'y' => $elem[2]]; + } + return $out; + } + +} \ No newline at end of file diff --git a/modules-available/vmstore/page.inc.php b/modules-available/vmstore/page.inc.php index 60a9d60a..41e7e990 100644 --- a/modules-available/vmstore/page.inc.php +++ b/modules-available/vmstore/page.inc.php @@ -11,6 +11,19 @@ class Page_VmStore extends Page { User::load(); + if (User::hasPermission('edit')) { + Dashboard::addSubmenu('?do=vmstore', Dictionary::translate('menu_edit', true)); + } + if (User::hasPermission('benchmark')) { + Dashboard::addSubmenu('?do=vmstore&show=benchmark', Dictionary::translate('menu_benchmark', true)); + } + + if (Request::any('show') === 'benchmark') { + User::assertPermission('benchmark'); + $this->benchmarkDoPreprocess(); + return; + } + User::assertPermission('edit'); $action = Request::post('action'); @@ -22,6 +35,11 @@ class Page_VmStore extends Page protected function doRender() { + if (Request::any('show') === 'benchmark') { + $this->benchmarkDoRender(); + return; + } + $action = Request::post('action'); if ($action === 'setstore' && !Taskmanager::isFailed(Taskmanager::status($this->mountTask))) { Render::addTemplate('mount', array( @@ -68,4 +86,192 @@ class Page_VmStore extends Page } } + private function benchmarkDoPreprocess() + { + if (!Module::isAvailable('rebootcontrol')) { + ErrorHandler::traceError('rebootcontrol module not enabled'); + } + if (Request::post('action') === 'start') { + $this->benchmarkActionStart(); + } + } + + private function benchmarkDoRender() + { + switch (Request::get('action')) { + case 'select': + $this->benchmarkShowImageSelect(); + break; + case 'result': + $this->benchmarkShowResult(); + } + } + + private function benchmarkActionStart() + { + Module::isAvailable('dnbd3'); + $id = Request::post('id', Request::REQUIRED, 'string'); + $data = Session::get('benchmark-' . $id); + if (!isset($data['machines'])) { + Message::addError('invalid-benchmark-job', $id); + return; + } + if (isset($data['task'])) { + if ($data['task'] === 'inprogress') { + // Let's hope the proper ID gets written in a short while + sleep(1); + } + Util::redirect('?do=vmstore&show=benchmark&action=result&id=' . $id); + } + $nfs = !Dnbd3::isEnabled(); + $data['image'] = Request::post('image', Request::REQUIRED, 'string'); + // Save once first to minimize race window + $data['task'] = 'inprogress'; + Session::set('benchmark-' . $id, $data, 60); + Session::saveExtraData(); + $start = 0; + $data['task'] = VmStoreBenchmark::start($id, $data['machines'], $data['image'], $nfs, $start); + if ($data['task'] === null) { + $data['task'] = 'failed'; + } else { + // Test is 2x 30 seconds + $data['expected'] = $start + 60; + } + error_log('Saving: ' . json_encode($data)); + Session::set('benchmark-' . $id, $data, 60); + Util::redirect('?do=vmstore&show=benchmark&action=result&id=' . $id); + } + + private function benchmarkShowImageSelect() + { + $id = Request::get('id', Request::REQUIRED, 'string'); + $data = Session::get('benchmark-' . $id); + if (!isset($data['machines'])) { + Message::addError('invalid-benchmark-job', $id); + return; + } + if (isset($data['task'])) { + Message::addWarning('benchmark-already-started'); + Util::redirect('?do=vmstore&show=benchmark&action=result&id=' . $id); + } + Module::isAvailable('dnbd3'); + $lookup = Dnbd3::getActiveServers(); + $list = Dnbd3Rpc::getStatsMulti(array_keys($lookup), [Dnbd3Rpc::QUERY_IMAGES]); + if (empty($list)) { + Message::addError('dnbd3-failed'); + Util::redirect('?do=vmstore'); + } + $images = []; + foreach ($list as $json) { + foreach ($json['images'] as $img) { + $name = $img['name'] . ':' . $img['rid']; + if (!isset($images[$name])) { + $images[$name] = [ + 'users' => 0, + 'size' => Util::readableFileSize($img['size'], 1), + 'name' => $name, + 'id' => count($images) + ]; + } + $images[$name]['users'] += $img['users']; + } + } + ArrayUtil::sortByColumn($images, 'users', SORT_NUMERIC | SORT_DESC); + Render::addTemplate('benchmark-imgselect', ['id' => $id, 'list' => array_values($images)]); + } + + private function benchmarkShowResult() + { + $id = Request::get('id', Request::REQUIRED, 'string'); + $data = Session::get('benchmark-' . $id); + if (!isset($data['machines'])) { + Message::addError('invalid-benchmark-job', $id); + return; + } + if (!isset($data['task'])) { + Message::addWarning('select-image-first'); + Util::redirect('?do=vmstore&show=benchmark&action=select&id=' . $id); + } + if ($data['task'] === 'failed') { + Message::addError('benchmark-failed'); + return; + } + $remaining = 0; + if ($data['task'] !== 'done') { + $remaining = $data['expected'] - time(); + if ($remaining < 0) { + $remaining = 0; + } + $this->processRunningBenchmark($id, $data, $remaining === 0); + $refresh = $remaining; + Util::clamp($refresh, 2, 64); + } + $args = [ + 'id' => $id, + 'result' => json_encode($data['result'] ?? []), + 'wanted' => json_encode($data['machines']), + ]; + if ($remaining > 0) { + $args['remaining'] = $remaining; + $args['refresh'] = $refresh; + } + Module::isAvailable('js_chart'); + Render::addTemplate('benchmark-result', $args); + } + + private function processRunningBenchmark(string $id, array &$data, bool $timeout) + { + Module::isAvailable('rebootcontrol'); + $changed = false; + + $active = array_filter($data['machines'], function ($e) use ($data) { return !isset($data['result'][$e]); }); + if (empty($active)) { + $timeout = true; + } else { + $command = << $out) { + if (isset($data['result'][$uuid])) + continue; + error_log(json_encode($out)); + // Not finished, ignore + if (($out['state'] !== 'DONE' || $out['exitCode'] !== 0) && !$timeout) + continue; + $changed = true; + unset($client); + $client = ['machineuuid' => $uuid]; + $data['result'][$uuid] =& $client; + if (preg_match_all("/^\+(\w{3}):(\d+),(.*)$/m", $out['stdout'], $modes, PREG_SET_ORDER)) { + foreach ($modes as $mode) { + $client[$mode[1]] = [ + 'start' => $mode[2], + 'values' => VmStoreBenchmark::parseBenchLine($mode[3]), + ]; + } + } + $m = Database::queryFirst('SELECT clientip, hostname FROM machine WHERE machineuuid = :uuid', + ['uuid' => $uuid]); + $client['name'] = empty($m['hostname']) ? $m['clientip'] : $m['hostname']; + } + } + if (count($data['result']) === count($data['machines']) || $timeout) { + $data['task'] = 'done'; + $changed = true; + } + if ($changed) { + Session::set('benchmark-' . $id, $data); + } + } + } \ No newline at end of file diff --git a/modules-available/vmstore/permissions/permissions.json b/modules-available/vmstore/permissions/permissions.json index 8303fd02..0617c673 100644 --- a/modules-available/vmstore/permissions/permissions.json +++ b/modules-available/vmstore/permissions/permissions.json @@ -1,5 +1,8 @@ { "edit": { "location-aware": false + }, + "benchmark": { + "location-aware": true } } \ No newline at end of file diff --git a/modules-available/vmstore/templates/benchmark-imgselect.html b/modules-available/vmstore/templates/benchmark-imgselect.html new file mode 100644 index 00000000..26ac898e --- /dev/null +++ b/modules-available/vmstore/templates/benchmark-imgselect.html @@ -0,0 +1,41 @@ +

{{lang_benchmark}}

+ +

{{lang_selectImage}}

+ +
+ + + + + + + + + + + + + + {{#list}} + + + + + + {{/list}} + +
{{lang_image}}{{lang_users}}{{lang_size}}
+
+ + +
+ 		{{users}}{{size}}
+ +
+ +
+ +
\ No newline at end of file diff --git a/modules-available/vmstore/templates/benchmark-result.html b/modules-available/vmstore/templates/benchmark-result.html new file mode 100644 index 00000000..fc7f8a55 --- /dev/null +++ b/modules-available/vmstore/templates/benchmark-result.html @@ -0,0 +1,106 @@ +

{{lang_benchmark}}

+ +

{{lang_benchmarkResult}}

+ +{{#remaining}} +
+ {{lang_benchmarkSecondsReminaing}}: {{remaining}} +
+{{/remaining}} + +
+ + \ No newline at end of file diff --git a/modules-available/vmstore/templates/page-vmstore.html b/modules-available/vmstore/templates/page-vmstore.html index 0e1ad601..fa222631 100644 --- a/modules-available/vmstore/templates/page-vmstore.html +++ b/modules-available/vmstore/templates/page-vmstore.html @@ -1,10 +1,11 @@ +

{{lang_vmLocation}}

+
-

{{lang_vmLocation}}

{{lang_vmLocationChoose}}

-- cgit v1.2.3-55-g7522