From 91990062dfea38aa92e4590579dd7e1e0d0ef7ec Mon Sep 17 00:00:00 2001 From: Jannik Schönartz Date: Wed, 24 Nov 2021 00:55:33 +0100 Subject: Add script for downloading/merging the edited image --- scripts/scp-snapshot | 101 +++++++++++++++++++++ .../org/openslx/taskmanager/tasks/ScpSnapshot.java | 96 ++++++++++++++++++++ 2 files changed, 197 insertions(+) create mode 100755 scripts/scp-snapshot create mode 100644 src/main/java/org/openslx/taskmanager/tasks/ScpSnapshot.java diff --git a/scripts/scp-snapshot b/scripts/scp-snapshot new file mode 100755 index 0000000..48afb9e --- /dev/null +++ b/scripts/scp-snapshot @@ -0,0 +1,101 @@ +#!/bin/bash + +# Working directory has to be owned by taskmanager +if [ ! -d /tmp/taskmanager ];then + mkdir /tmp/taskmanger +fi +workdir=/tmp/taskmanager + +# For the service, the ssh keys has to be in the /home/taskmanager/.ssh directory +# identityFile=/root/.ssh/ssh_poolclients +identityFile=~/.ssh/ssh_poolclients + +# Info for loggin into the client +# Always login as root for now but $TM_USERNAME can be set via the apicall in slx-admin &username=XXX +user=root +UUID=$TM_UUID + +# Clientip is set during the slx-admin api call clientip=XXX +client=$TM_CLIENT_IP + +filePath=/tmp/upload/$UUID.qcow2 +fileName=$(basename $filePath) + +# Write ssh fingerprint in known_hosts file +ssh-keyscan -H $client >> ~/.ssh/known_hosts + +# Get filesize via scp +fileSize=$(ssh -q -i $identityFile $user@$client "stat --printf='%s' $filePath") +fileDestination=$workdir/snapshot_$UUID.qcow2 + +script -q -f -c "scp -i $identityFile $user@$client:$filePath $fileDestination" $workdir/typescript > $workdir/progress & + +WEBSUITE_BACKEND="" +lastProgress="" +while true +do + # Get the last line written by the scp process + progress=$(tail -n 1 $workdir/progress | tr " " "$" | rev | cut -d "$" -f2 | rev) + + # Ignore when the progress didn't change + if [ "$lastProgress" == "$progress" ]; then + continue + fi + + # If status did change, cut the % from the scp progress and see if it's 100% (finished) + percent=$(echo $progress | cut -d " " -f2) + size=$(echo $progress | cut -d " " -f3) + speed=$(echo $progress | cut -d " " -f4) + eta=$(echo $progress | cut -d " " -f5) + + # Upload the progress to the bwLehrpool webSuite backend + curl -X POST https://$WEBSUITE_BACKEND/images/$UUID/versions/upload/progress -H "Content-Type: application/json" -d '{ "fileName": '$fileName', "progress": '$percent', "size": '$size', "speed": '$speed', "eta": '$eta' }' + + if [ "$percent" == "100%" ]; then + break + fi + + # Set the lastProgess to the current progress and goto the next iteration of the loop + lastProgress=$progress +done + +# Upload is finished so delete snapshot on client +delete=$(ssh -q -i $identityFile $user@$client "rm $filePath") + +# DB variables +db_user="" +db_password="" + +# Merge snapshot with image +# 1. Make a copy of the baseimage with rsync (to have progress) +image_version_uuid=$(mysql -u $db_user --password=$db_password sat -se "SELECT imageversionid from lecture WHERE lectureid='$UUID'") +image_path=$(mysql -u $db_user --password=$db_password sat -se "SELECT filepath from imageversion WHERE imageversionid='$image_version_uuid'") +# TODO: Currently only nfs? +bwlp_store=/srv/openslx/nfs +image_base_copy="$(date +'%y-%m')/$(date +'%d_%H-%M-%S_merged-image.qcow2')" +script -q -f -c "rsync --mkpath --info=progress2 $bwlp_store/$image_path $bwlp_store/bwlehrpool_store/$image_base_copy" $workdir/typescript_copy > $workdir/progress_copy #& +# TODO while true read progress until 100 look above! + +# 2. Rebase the snapshot -- use -u (unsafe) so it works LUL +qemu-img rebase -u -b "$bwlp_store/bwlehrpool_store/$image_base_copy" $fileDestination + +# 3. Commit (merge) the snapshot with the baseimage +qemu-img commit $fileDestination + +# 4. Update database entries, to set the new image to the newest version +image_expire=$(mysql -u $db_user --password=$db_password sat -se "SELECT expiretime from imageversion WHERE imageversionid='$image_version_uuid'") +image_base_uuid=$(mysql -u $db_user --password=$db_password sat -se "SELECT imagebaseid from imageversion WHERE imageversionid='$image_version_uuid'") +image_filesize=$(mysql -u $db_user --password=$db_password sat -se "SELECT filesize from imageversion WHERE imageversionid='$image_version_uuid'") +image_uuid=$(cat /proc/sys/kernel/random/uuid) +image_virtualizerconfig=$(mysql -u $db_user --password=$db_password sat -se "SELECT virtualizerconfig from imageversion WHERE imageversionid='$image_version_uuid'") + +insert_query="INSERT INTO imageversion (imageversionid, imagebaseid, createtime, expiretime, filesize, filepath, uploaderid, virtualizerconfig, isrestricted, isvalid, isprocessed) VALUES ('$image_uuid','$image_base_uuid',$(date +%s),$image_expire,$image_filesize,'bwlehrpool_store/$image_base_copy','af73720a8be1aace58debedc21d7cb30','$image_virtualizerconfig',0,1,0);" +update_query="UPDATE imagebase SET latestversionid = '$image_uuid' WHERE imagebaseid = '$image_base_uuid';" + +mysql -u $db_user --password=$db_password sat -e "$insert_query" +mysql -u $db_user --password=$db_password sat -e "$update_query" + +# Delete downloaded /tmp/taskmanager stuff +rm -rf /tmp/taskmanager/* + +exit 0 diff --git a/src/main/java/org/openslx/taskmanager/tasks/ScpSnapshot.java b/src/main/java/org/openslx/taskmanager/tasks/ScpSnapshot.java new file mode 100644 index 0000000..0fe0c24 --- /dev/null +++ b/src/main/java/org/openslx/taskmanager/tasks/ScpSnapshot.java @@ -0,0 +1,96 @@ +package org.openslx.taskmanager.tasks; + +import java.util.Map; + +import org.openslx.satserver.util.Constants; +import org.openslx.taskmanager.api.SystemCommandTask; + +import com.google.gson.annotations.Expose; + +public class ScpSnapshot extends SystemCommandTask +{ + @Expose + private String lectureUuid = null; + @Expose + private String username = null; + @Expose + private String clientIp = null; + @Expose + private String snapshotPath = null; + + private final Output status = new Output(); + + + @Override + protected boolean initTask() + { + this.setStatusObject( this.status ); + if ( this.lectureUuid == null ) + this.lectureUuid = ""; + if ( this.username == null ) + this.username = ""; + if ( this.clientIp == null ) + this.clientIp = ""; + if ( this.snapshotPath == null ) + this.snapshotPath = ""; + + return true; + } + + @Override + protected String[] initCommandLine() + { + return new String[] { + "/usr/bin/sudo", + "-n", + "-u", "root", + // "/bin/bash", + Constants.BASEDIR + "/scripts/scp-snapshot" + }; + } + + @Override + protected void initEnvironment( Map environment ) + { + environment.put( "TM_UUID", this.lectureUuid ); + environment.put( "TM_USERNAME", this.username); + environment.put( "TM_CLIENT_IP", this.clientIp); + environment.put( "TM_SNAPSHOT_PATH", this.snapshotPath); + } + + @Override + protected boolean processEnded( int exitCode ) + { + return exitCode == 0; + } + + @Override + protected void processStdOut( String line ) + { + status.addMessage( line ); + } + + @Override + protected void processStdErr( String line ) + { + status.addMessage( line ); + } + + /** + * Output - contains additional status data of this task + */ + class Output + { + private String messages = null; + + private void addMessage( String str ) + { + if ( messages == null ) { + messages = str; + } else { + messages += "\n" + str; + } + } + } + +} -- cgit v1.2.3-55-g7522