[server] Store block hashes in DB

7 files changed, 176 insertions, 41 deletions

diff --git a/dozentenmodulserver/setup/sat-01-schema.sql b/dozentenmodulserver/setup/sat-01-schema.sql
index 051ab485..f030298a 100644
--- a/dozentenmodulserver/setup/sat-01-schema.sql
+++ b/dozentenmodulserver/setup/sat-01-schema.sql
@@ -74,7 +74,7 @@ CREATE TABLE IF NOT EXISTS `imageversion` (
   `isvalid` tinyint(1) NOT NULL,
   `isprocessed` tinyint(1) NOT NULL,
   `mastersha1` binary(20) DEFAULT NULL,
-  `virtualizerconfig` text BINARY NULL DEFAULT NULL COMMENT 'Specific configuration of the virtualizer for this image. For vmware, this is basically a dump of the *.vmx.',
+  `virtualizerconfig` blob NULL DEFAULT NULL COMMENT 'Specific configuration of the virtualizer for this image. For vmware, this is basically a dump of the *.vmx.',
   PRIMARY KEY (`imageversionid`),
   KEY `version_access` (`imagebaseid`,`isvalid`,`createtime`),
   KEY `fk_imageversion_2_idx` (`uploaderid`),
diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbImage.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbImage.java
index c8e5fd2a..8a10fb85 100644
--- a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbImage.java
+++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbImage.java
@@ -1,6 +1,7 @@
 package org.openslx.bwlp.sat.database.mappers;
 
 import java.io.File;
+import java.nio.ByteBuffer;
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.util.ArrayList;
@@ -13,6 +14,7 @@ import org.openslx.bwlp.sat.database.Database;
 import org.openslx.bwlp.sat.database.MysqlConnection;
 import org.openslx.bwlp.sat.database.MysqlStatement;
 import org.openslx.bwlp.sat.database.Paginator;
+import org.openslx.bwlp.sat.database.models.ImageVersionMeta;
 import org.openslx.bwlp.sat.database.models.LocalImageVersion;
 import org.openslx.bwlp.sat.mail.MailGenerator;
 import org.openslx.bwlp.sat.permissions.User;
@@ -29,6 +31,7 @@ import org.openslx.bwlp.thrift.iface.ShareMode;
 import org.openslx.bwlp.thrift.iface.TNotFoundException;
 import org.openslx.bwlp.thrift.iface.UserInfo;
 import org.openslx.filetransfer.util.ChunkList;
+import org.openslx.filetransfer.util.FileChunk;
 import org.openslx.util.QuickTimer;
 import org.openslx.util.QuickTimer.Task;
 
@@ -103,6 +106,7 @@ public class DbImage {
 					rs.getLong("createtime"), rs.getLong("updatetime"), rs.getString("ownerid"),
 					rs.getString("updaterid"), toShareMode(rs.getString("sharemode")),
 					rs.getByte("istemplate") != 0, defaultPermissions);
+			image.setUserPermissions(DbImagePermissions.fromResultSetUser(rs));
 			User.setCombinedUserPermissions(image, user);
 			return image;
 		} catch (SQLException e) {
@@ -536,16 +540,11 @@ public class DbImage {
 			verStmt.setBinary("mastersha1", null); // TODO
 			verStmt.setBinary("virtualizerconfig", machineDescription);
 			verStmt.executeUpdate();
-			// TODO: Write chunk hashes to DB
-			// Make this version the latest version
-			MysqlStatement baseStmt = connection.prepareStatement("UPDATE imagebase SET"
-					+ " latestversionid = :imageversionid WHERE imagebaseid = :imagebaseid LIMIT 1");
-			baseStmt.setString("imageversionid", imageVersionId);
-			baseStmt.setString("imagebaseid", imageBaseId);
-			baseStmt.executeUpdate();
+			writeChunks(connection, imageVersionId, chunks);
 			LocalImageVersion liv = new LocalImageVersion(imageVersionId, imageBaseId, filePath, fileSize,
 					owner.userId, nowSecs, expireTime, true);
 			DbLecture.autoUpdateUsedImage(connection, imageBaseId, liv);
+			// Make this version the latest version
 			setLatestVersion(connection, imageBaseId, liv);
 			connection.commit();
 		} catch (SQLException e) {
@@ -554,6 +553,26 @@ public class DbImage {
 		}
 	}
 
+	private static void writeChunks(MysqlConnection connection, String imageVersionId, ChunkList chunks)
+			throws SQLException {
+		if (chunks == null || chunks.isEmpty())
+			return;
+		for (FileChunk chunk : chunks.getAll()) {
+			if (chunk.getSha1Sum() == null)
+				return;
+		}
+		MysqlStatement stmt = connection.prepareStatement("INSERT IGNORE INTO imageblock"
+				+ " (imageversionid, startbyte, blocksize, blocksha1, ismissing) VALUES"
+				+ " (:imageversionid, :startbyte, :blocksize, :blocksha1, 0)");
+		stmt.setString("imageversionid", imageVersionId);
+		for (FileChunk chunk : chunks.getAll()) {
+			stmt.setLong("startbyte", chunk.range.startOffset);
+			stmt.setInt("blocksize", chunk.range.getLength());
+			stmt.setBinary("blocksha1", chunk.getSha1Sum());
+			stmt.executeUpdate();
+		}
+	}
+
 	protected static void markValid(MysqlConnection connection, boolean valid,
 			LocalImageVersion... imageVersion) throws SQLException {
 		if (imageVersion == null || imageVersion.length == 0)
@@ -781,4 +800,35 @@ public class DbImage {
 		}
 	}
 
+	public static ImageVersionMeta getVersionDetails(String imageVersionId) throws SQLException,
+			TNotFoundException {
+		try (MysqlConnection connection = Database.getConnection()) {
+			MysqlStatement stmt = connection.prepareStatement("SELECT"
+					+ " imageversionid, imagebaseid, virtualizerconfig FROM imageversion"
+					+ " WHERE imageversionid = :imageversionid");
+			stmt.setString("imageversionid", imageVersionId);
+			ResultSet rs = stmt.executeQuery();
+			if (!rs.next())
+				throw new TNotFoundException();
+			return new ImageVersionMeta(imageVersionId, rs.getString("imagebaseid"),
+					rs.getBytes("virtualizerconfig"), getBlockHashes(connection, imageVersionId));
+		} catch (SQLException e) {
+			LOGGER.error("Query failed in DbImage.getVersionDetails()", e);
+			throw e;
+		}
+	}
+
+	private static List<ByteBuffer> getBlockHashes(MysqlConnection connection, String imageVersionId)
+			throws SQLException {
+		MysqlStatement stmt = connection.prepareStatement("SELECT blocksha1 FROM imageblock"
+				+ " WHERE imageversionid = :imageversionid ORDER BY startbyte ASC");
+		stmt.setString("imageversionid", imageVersionId);
+		ResultSet rs = stmt.executeQuery();
+		List<ByteBuffer> list = new ArrayList<>();
+		while (rs.next()) {
+			list.add(ByteBuffer.wrap(rs.getBytes("blocksha1")));
+		}
+		return list;
+	}
+
 }
diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/fileserv/IncomingDataTransfer.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/fileserv/IncomingDataTransfer.java
index e4e3349e..d008f5db 100644
--- a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/fileserv/IncomingDataTransfer.java
+++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/fileserv/IncomingDataTransfer.java
@@ -4,7 +4,6 @@ import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.RandomAccessFile;
-import java.nio.charset.StandardCharsets;
 import java.security.NoSuchAlgorithmException;
 import java.sql.SQLException;
 import java.util.ArrayList;
@@ -156,7 +155,7 @@ public class IncomingDataTransfer extends AbstractTransfer implements HashCheckC
 		this.owner = publishData.user;
 		this.image = idr;
 		this.fileSize = publishData.fileSize;
-		this.machineDescription = transferInfo.machineDescription.getBytes(StandardCharsets.UTF_8);
+		this.machineDescription = ThriftUtil.unwrapByteBuffer(transferInfo.machineDescription);
 		this.masterTransferInfo = transferInfo;
 		this.versionSettings = new ImageVersionWrite(false);
 	}
diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/mail/MailGenerator.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/mail/MailGenerator.java
index 47006742..2a47f5e8 100644
--- a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/mail/MailGenerator.java
+++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/mail/MailGenerator.java
@@ -1,9 +1,11 @@
 package org.openslx.bwlp.sat.mail;
 
 import java.sql.SQLException;
+import java.text.BreakIterator;
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
+import java.util.Locale;
 import java.util.Map;
 import java.util.Map.Entry;
 
@@ -50,7 +52,7 @@ public class MailGenerator {
 			for (UserInfo user : relevantUsers) {
 				if (newVersion.uploaderId.equals(user.userId)) // Don't notice about changes by user
 					continue;
-				MailQueue.queue(new Mail(user, message));
+				MailQueue.queue(new Mail(user, wordWrap(message)));
 			}
 		}
 	}
@@ -67,13 +69,13 @@ public class MailGenerator {
 			return;
 		for (LectureSummary lecture : lectures) {
 			List<UserInfo> relevantUsers = getUserToMail(lecture);
-			String message = "Die VM-Version der Veranstaltung '" + lecture.lectureName
-					+ "' wurde gelöscht oder ist beschädigt,\n"
-					+ "   daher verweist sie jetzt auf die VM-Version vom "
-					+ Formatter.date(newVersion.createTime) + ".\n"
-					+ "   Bitte überprüfen Sie ggf. die Verlinkte VM bzgl. Ihrer Kursinhalte.";
+			String message = "Die verlinkte VM zur Veranstaltung '" + lecture.lectureName
+					+ "' wurde gelöscht oder ist beschädigt,"
+					+ " daher verweist sie jetzt auf die VM-Version vom "
+					+ Formatter.date(newVersion.createTime) + "."
+					+ " Bitte überprüfen Sie ggf., ob diese VM-Version für Ihren Kurs geeignet ist.";
 			for (UserInfo user : relevantUsers) {
-				MailQueue.queue(new Mail(user, message));
+				MailQueue.queue(new Mail(user, wordWrap(message)));
 			}
 		}
 	}
@@ -84,11 +86,11 @@ public class MailGenerator {
 		for (LectureSummary lecture : lectures) {
 			List<UserInfo> relevantUsers = getUserToMail(lecture);
 			String message = "Die Veranstaltung '" + lecture.lectureName + "' musste deaktiviert werden,"
-					+ "\n   da die verknüpfte VM gelöscht oder beschädigt wurde. Bitte prüfen"
-					+ "\n   Sie die Veranstaltung und ändern Sie ggf. die Verlinkung,"
-					+ "\n   damit die Veranstaltung nicht gelöscht wird.";
+					+ " da die verknüpfte VM gelöscht oder beschädigt wurde. Bitte überprüfen"
+					+ " Sie die Veranstaltung und ändern Sie ggf. die Verlinkung,"
+					+ " damit die Veranstaltung wieder verwendbar ist.";
 			for (UserInfo user : relevantUsers) {
-				MailQueue.queue(new Mail(user, message));
+				MailQueue.queue(new Mail(user, wordWrap(message)));
 			}
 		}
 	}
@@ -114,27 +116,26 @@ public class MailGenerator {
 				newVersion = version;
 			}
 		}
-		String message = "VM '" + image.imageName + "':";
+		String message = "Virtuelle Maschine '" + image.imageName + "':";
 		if (newVersion == null) {
-			message += "\n   Die letzte verbliebene Version der VM wurde gelöscht; VM deaktiviert.";
+			message += " Die letzte verbliebene Version der VM wurde gelöscht; VM deaktiviert.";
 		} else {
 			String uploaderName;
 			try {
 				User uploader = DbUser.getCached(newVersion.uploaderId);
-				uploaderName = uploader.ui.firstName + " " + uploader.ui.lastName + " <" + uploader.ui.eMail
-						+ ">";
+				uploaderName = Formatter.userFullName(uploader.ui) + " <" + uploader.ui.eMail + ">";
 			} catch (TNotFoundException | SQLException e) {
 				uploaderName = "(unbekannt)";
 			}
-			message += "\n   Neueste Version ist jetzt vom " + Formatter.date(oldVersion.createTime)
+			message += "\n  Neueste Version ist jetzt vom " + Formatter.date(oldVersion.createTime)
 					+ " (erstellt von " + uploaderName + ")";
 			if (oldVersion != null) {
-				message += "\n   Zuvor neueste Verson war vom " + Formatter.date(oldVersion.createTime);
+				message += "\n  Vorherige Verson war vom " + Formatter.date(oldVersion.createTime);
 			}
 		}
 		List<UserInfo> relevantUsers = getUserToMail(image);
 		for (UserInfo user : relevantUsers) {
-			MailQueue.queue(new Mail(user, message));
+			MailQueue.queue(new Mail(user, wordWrap(message)));
 		}
 	}
 
@@ -150,18 +151,19 @@ public class MailGenerator {
 		}
 		String message;
 		if (image.latestVersionId == null || image.latestVersionId.equals(version.imageVersionId)) {
-			message = "Die aktuellste Version der VM '" + image.imageName + "' läuft\n" + "   in " + days
-					+ " Tag(en) ab. Bitte aktualisieren Sie die VM.";
+			message = "Die aktuellste Version der VM '" + image.imageName + "' läuft in " + days
+					+ " Tag(en) ab. Bitte aktualisieren Sie die VM, da verknüpfte"
+					+ " Veranstaltungen sonst deaktiviert werden.";
 		} else if (mailForced) {
 			message = "Eine alte Version der VM '" + image.imageName + "' läuft in " + days
-					+ " Tag(en) ab.\n"
-					+ "   Eine aktuellere Version ist vorhanden, diese Nachricht ist rein informell.";
+					+ " Tag(en) ab (Version vom " + Formatter.date(version.createTime) + ")."
+					+ " Eine aktuellere Version ist vorhanden, diese Nachricht dient nur der Information.";
 		} else {
 			return;
 		}
 		List<UserInfo> relevantUsers = getUserToMail(image);
 		for (UserInfo user : relevantUsers) {
-			MailQueue.queue(new Mail(user, message));
+			MailQueue.queue(new Mail(user, wordWrap(message)));
 		}
 	}
 
@@ -171,7 +173,7 @@ public class MailGenerator {
 		List<UserInfo> relevantUsers = getUserToMail(lecture);
 		String message = "Die Veranstaltung '" + lecture.lectureName + "' läuft in " + days + " Tag(en) ab.";
 		for (UserInfo user : relevantUsers) {
-			MailQueue.queue(new Mail(user, message));
+			MailQueue.queue(new Mail(user, wordWrap(message)));
 		}
 	}
 
@@ -248,4 +250,69 @@ public class MailGenerator {
 		return list;
 	}
 
+	private static String wordWrap(String input) {
+		return wordWrap(input, 76, "\n  ", Locale.GERMAN);
+	}
+
+	private static String wordWrap(String input, int width, String nlString, Locale locale) {
+		if (input == null) {
+			return "";
+		} else if (width < 5) {
+			return input;
+		} else if (width >= input.length()) {
+			return input;
+		}
+
+		StringBuilder buf = new StringBuilder(input);
+		boolean endOfLine = false;
+		int lineStart = 0;
+
+		for (int i = 0; i < buf.length(); i++) {
+			if (buf.charAt(i) == '\n') {
+				lineStart = i + 1;
+				endOfLine = true;
+			}
+
+			// handle splitting at width character
+			if (i > lineStart + width - 1) {
+				if (endOfLine) {
+					buf.insert(i, nlString);
+					lineStart = i + nlString.length();
+					endOfLine = false;
+				} else {
+					int limit = i - lineStart - 1;
+					BreakIterator breaks = BreakIterator.getLineInstance(locale);
+					breaks.setText(buf.substring(lineStart, i));
+					int end = breaks.last();
+
+					// if the last character in the search string isn't a space,
+					// we can't split on it (looks bad). Search for a previous
+					// break character
+					if (end == limit + 1) {
+						if (!Character.isWhitespace(buf.charAt(lineStart + end))) {
+							end = breaks.preceding(end - 1);
+						}
+					}
+
+					// if the last character is a space, replace it with a \n
+					if (end != BreakIterator.DONE && end == limit + 1) {
+						buf.replace(lineStart + end, lineStart + end + 1, nlString.substring(0, 1));
+						buf.insert(lineStart + end + 1, nlString.substring(1));
+						lineStart = lineStart + end + nlString.length() - 1;
+					}
+					// otherwise, just insert a \n
+					else if (end != BreakIterator.DONE && end != 0) {
+						buf.insert(lineStart + end, nlString);
+						lineStart = lineStart + end + nlString.length();
+					} else {
+						buf.insert(i, nlString);
+						lineStart = i + nlString.length();
+					}
+				}
+			}
+		}
+
+		return buf.toString();
+	}
+
 }
diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/mail/MailQueue.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/mail/MailQueue.java
index 0ba396d2..6ed921ff 100644
--- a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/mail/MailQueue.java
+++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/mail/MailQueue.java
@@ -184,7 +184,7 @@ public class MailQueue {
 		}, TimeUnit.MINUTES.toMillis(1));
 	}
 
-	private static boolean sendMail(MailConfig conf, SmtpMailer smtpc, UserInfo user, String message) {
+	private static boolean sendMail(MailConfig conf, SmtpMailer smtpc, UserInfo user, final String message) {
 		// TODO: Template
 		String fullMessage = "Guten Tag " + user.firstName + " " + user.lastName + ",\n\n"
 				+ "Bitte beachten Sie folgende Hinweise zu Virtuellen Maschinen und Veranstaltungen,\n"
@@ -199,8 +199,14 @@ public class MailQueue {
 			fullMessage += "\n\nBei weiteren Fragen wenden Sie sich bitte an den Support unter\n"
 					+ conf.replyTo;
 		}
-		fullMessage += "\n\n--\n" + "Generiert auf " + conf.serverName;
-		return smtpc.send(user.eMail, "[bwLehrstuhl] Hinweise zu Ihren VMs/Veranstaltungen", fullMessage);
+		fullMessage += "\n\n-- \n" + "Generiert auf " + conf.serverName;
+		if (fullMessage.contains("\r\n")) {
+			fullMessage = fullMessage.replace("\r\n", "\n");
+		}
+		if (fullMessage.contains("\n")) {
+			fullMessage = fullMessage.replace("\n", "\r\n");
+		}
+		return smtpc.send(user.eMail, "[bwLehrpool] Hinweise zu Ihren VMs/Veranstaltungen", fullMessage);
 	}
 
 }
diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/permissions/User.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/permissions/User.java
index 0fc8a365..191a5f92 100644
--- a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/permissions/User.java
+++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/permissions/User.java
@@ -183,11 +183,14 @@ public class User {
 				"No permission to delete this image");
 	}
 
-	public static void canDownloadImageVersionOrFail(UserInfo user, String imageVersionId)
+	public static void canDownloadImageVersionOrFail(UserInfo user, String imageBaseId, String imageVersionId)
 			throws TAuthorizationException, TNotFoundException, TInvocationException {
 		ImageDetailsRead image;
 		try {
-			image = DbImage.getImageDetails(user, DbImage.getBaseIdForVersionId(imageVersionId));
+			if (imageBaseId == null) {
+				imageBaseId = DbImage.getBaseIdForVersionId(imageVersionId);
+			}
+			image = DbImage.getImageDetails(user, imageBaseId);
 		} catch (SQLException e) {
 			throw new TInvocationException();
 		}
diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/thrift/ServerHandler.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/thrift/ServerHandler.java
index 3b76efcd..94d95042 100644
--- a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/thrift/ServerHandler.java
+++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/thrift/ServerHandler.java
@@ -13,6 +13,7 @@ import org.openslx.bwlp.sat.database.mappers.DbImagePermissions;
 import org.openslx.bwlp.sat.database.mappers.DbLecture;
 import org.openslx.bwlp.sat.database.mappers.DbLecturePermissions;
 import org.openslx.bwlp.sat.database.mappers.DbUser;
+import org.openslx.bwlp.sat.database.models.ImageVersionMeta;
 import org.openslx.bwlp.sat.database.models.LocalUser;
 import org.openslx.bwlp.sat.fileserv.FileServer;
 import org.openslx.bwlp.sat.fileserv.IncomingDataTransfer;
@@ -133,14 +134,23 @@ public class ServerHandler implements SatelliteServer.Iface {
 			throws TAuthorizationException, TInvocationException, TNotFoundException,
 			TTransferRejectedException {
 		UserInfo user = SessionManager.getOrFail(userToken);
-		User.canDownloadImageVersionOrFail(user, imageVersionId);
+		ImageVersionMeta imageVersion;
+		try {
+			imageVersion = DbImage.getVersionDetails(imageVersionId);
+		} catch (SQLException e) {
+			throw new TInvocationException();
+		}
+		User.canDownloadImageVersionOrFail(user, imageVersion.imageBaseId, imageVersionId);
 		OutgoingDataTransfer transfer;
 		try {
 			transfer = fileServer.createNewUserDownload(DbImage.getLocalImageData(imageVersionId));
 		} catch (SQLException e) {
 			throw new TInvocationException();
 		}
-		return new TransferInformation(transfer.getId(), fileServer.getPlainPort(), fileServer.getSslPort());
+		TransferInformation ti = new TransferInformation(transfer.getId(), fileServer.getPlainPort(), fileServer.getSslPort());
+		ti.setBlockHashes(imageVersion.sha1sums);
+		ti.setMachineDescription(imageVersion.machineDescription);
+		return ti;
 	}
 
 	@Override