[server] Handle deletion/undeletion flags

10 files changed, 247 insertions, 15 deletions

diff --git a/dozentenmodulserver/src/main/java/fi/iki/elonen/NanoHTTPD.java b/dozentenmodulserver/src/main/java/fi/iki/elonen/NanoHTTPD.java
index 7296abea..ff552e83 100644
--- a/dozentenmodulserver/src/main/java/fi/iki/elonen/NanoHTTPD.java
+++ b/dozentenmodulserver/src/main/java/fi/iki/elonen/NanoHTTPD.java
@@ -953,7 +953,7 @@ public abstract class NanoHTTPD implements Runnable {
 					} catch (Exception e) {
 					}
 				}
-				LOGGER.debug("Content type is '" + contentType + "', encoding '" + cs.name() + "'");
+				//LOGGER.debug("Content type is '" + contentType + "', encoding '" + cs.name() + "'");
 
 				if ("multipart/form-data".equalsIgnoreCase(contentType)) {
 					throw new ResponseException(Response.Status.BAD_REQUEST,
diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/App.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/App.java
index 2648310b..93f3a0d1 100644
--- a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/App.java
+++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/App.java
@@ -112,6 +112,9 @@ public class App {
 			return;
 		}
 
+		// Start watch dog to ensure nobody else is messing with the vmstore
+		QuickTimer.scheduleAtFixedDelay(new StorageUseCheck(), 10000, 60000);
+
 		// Set up maintenance tasks
 		DeleteOldImages.init();
 		SendExpireWarning.init();
diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/StorageUseCheck.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/StorageUseCheck.java
new file mode 100644
index 00000000..a91e6535
--- /dev/null
+++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/StorageUseCheck.java
@@ -0,0 +1,73 @@
+package org.openslx.bwlp.sat;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.UUID;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.log4j.Logger;
+import org.openslx.bwlp.sat.util.Configuration;
+import org.openslx.bwlp.sat.util.FileSystem;
+import org.openslx.util.QuickTimer.Task;
+
+/**
+ * As it might have catastrophic results if two instances of this server operate
+ * on the same file storage, we write a randomly generated id to a file on the
+ * storage, and check it periodically. If it changed, we bail out in complete
+ * panic.
+ */
+public class StorageUseCheck extends Task {
+
+	private static final Logger LOGGER = Logger.getLogger(StorageUseCheck.class);
+
+	private final String uuid = UUID.randomUUID().toString();
+
+	private final File canary = new File(Configuration.getVmStoreProdPath(), "dozmod.lock");
+
+	private boolean created = false;
+
+	public StorageUseCheck() {
+		if (FileSystem.waitForStorage()) {
+			createCanary();
+		}
+	}
+
+	private void createCanary() {
+		if (!FileSystem.isStorageMounted()) {
+			LOGGER.warn("Cannot check storage lock, storage not mounted");
+			return;
+		}
+		if (!created || !canary.exists()) {
+			try {
+				FileUtils.write(canary, uuid);
+			} catch (IOException e) {
+				LOGGER.fatal("Cannot write lock file to VMStore", e);
+				System.exit(1);
+			}
+			created = true;
+		} else {
+			String canaryContents;
+			try {
+				canaryContents = FileUtils.readFileToString(canary);
+			} catch (IOException e) {
+				LOGGER.warn("Lock file cannot be accessed. Cannot ensure exclusive use of VMStore", e);
+				return;
+			}
+			if (!canaryContents.equals(uuid)) {
+				LOGGER.fatal("Lock file content changed. Another server instance is using the VMStore."
+						+ " Will exit immediately to prevent any damages.");
+				try {
+					FileUtils.write(canary, uuid);
+				} catch (IOException e) {
+				}
+				System.exit(1);
+			}
+		}
+	}
+
+	@Override
+	public void fire() {
+		createCanary();
+	}
+
+}
diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbImage.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbImage.java
index 2bf26208..7f12a5f6 100644
--- a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbImage.java
+++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbImage.java
@@ -5,7 +5,9 @@ import java.nio.ByteBuffer;
 import java.sql.ResultSet;
 import java.sql.SQLException;
 import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 import java.util.UUID;
 
 import org.apache.log4j.Logger;
@@ -514,7 +516,7 @@ public class DbImage {
 			MysqlStatement stmt = connection.prepareStatement("UPDATE imagebase"
 					+ " SET sharemode = :sharemode WHERE imagebaseid = :baseid LIMIT 1");
 			stmt.setString("baseid", imageBaseId);
-			stmt.setString("sharemode", newData.shareMode.toString());
+			stmt.setString("sharemode", newData.shareMode.name());
 			stmt.executeUpdate();
 			connection.commit();
 		} catch (SQLException e) {
@@ -550,7 +552,7 @@ public class DbImage {
 			verStmt.executeUpdate();
 			writeChunks(connection, imageVersionId, chunks);
 			LocalImageVersion liv = new LocalImageVersion(imageVersionId, imageBaseId, filePath, fileSize,
-					owner.userId, nowSecs, expireTime, true, DeleteState.KEEP.toString());
+					owner.userId, nowSecs, expireTime, true, DeleteState.KEEP.name());
 			DbLecture.autoUpdateUsedImage(connection, imageBaseId, liv);
 			// Update edit timestamp and edit user
 			MysqlStatement baseStmt = connection.prepareStatement("UPDATE imagebase SET"
@@ -912,14 +914,14 @@ public class DbImage {
 			return;
 		String ignoredOldState;
 		if (shouldDelete == DeleteState.SHOULD_DELETE) {
-			ignoredOldState = DeleteState.DO_DELETE.toString();
+			ignoredOldState = DeleteState.DO_DELETE.name();
 		} else {
 			ignoredOldState = "invalid";
 		}
 		try (MysqlConnection connection = Database.getConnection()) {
 			MysqlStatement stmt = connection.prepareStatement("UPDATE imageversion SET deletestate = :newstate"
 					+ " WHERE imageversionid = :imageversionid AND deletestate <> :oldstate");
-			stmt.setString("newstate", shouldDelete.toString());
+			stmt.setString("newstate", shouldDelete.name());
 			stmt.setString("oldstate", ignoredOldState);
 			for (String imageVersionId : imageVersionIds) {
 				if (imageVersionId == null)
@@ -938,7 +940,7 @@ public class DbImage {
 		try (MysqlConnection connection = Database.getConnection()) {
 			MysqlStatement stmt = connection.prepareStatement(localImageBaseSql
 					+ " WHERE deletestate = :deletestate");
-			stmt.setString("deletestate", state.toString());
+			stmt.setString("deletestate", state.name());
 			ResultSet rs = stmt.executeQuery();
 			List<LocalImageVersion> list = new ArrayList<>();
 			while (rs.next()) {
@@ -964,4 +966,37 @@ public class DbImage {
 		}
 	}
 
+	/**
+	 * Reset all image versions where the server decided that they should be
+	 * deleted to the 'keep' state.
+	 * 
+	 * @return list of version ids that were reset
+	 * 
+	 * @throws SQLException
+	 */
+	public static Set<String> resetDeleteState() throws SQLException {
+		try (MysqlConnection connection = Database.getConnection()) {
+			// Get
+			MysqlStatement sstmt = connection.prepareStatement("SELECT imageversionid FROM imageversion"
+					+ " WHERE deletestate = :should");
+			sstmt.setString("should", DeleteState.SHOULD_DELETE.name());
+			ResultSet rs = sstmt.executeQuery();
+			Set<String> list = new HashSet<>();
+			while (rs.next()) {
+				list.add(rs.getString("imageversionid"));
+			}
+			// Update
+			MysqlStatement ustmt = connection.prepareStatement("UPDATE imageversion SET deletestate = :keep"
+					+ " WHERE deletestate = :should");
+			ustmt.setString("keep", DeleteState.KEEP.name());
+			ustmt.setString("should", DeleteState.SHOULD_DELETE.name());
+			ustmt.executeUpdate();
+			connection.commit();
+			return list;
+		} catch (SQLException e) {
+			LOGGER.error("Query failed in DbImage.resetDeleteState()", e);
+			throw e;
+		}
+	}
+
 }
diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/fileserv/FileServer.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/fileserv/FileServer.java
index e74852d0..0e62d9c8 100644
--- a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/fileserv/FileServer.java
+++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/fileserv/FileServer.java
@@ -78,10 +78,10 @@ public class FileServer implements IncomingEvent {
 	@Override
 	public void incomingDownloadRequest(Uploader uploader) throws IOException {
 		String token = uploader.getToken();
-		LOGGER.info("Incoming filetransfer with token " + token);
+		LOGGER.debug("Incoming filetransfer with token " + token);
 		OutgoingDataTransfer download = downloads.get(token);
 		if (download == null) {
-			LOGGER.warn("Unknown token " + token);
+			LOGGER.warn("Download request: Unknown token " + token);
 			uploader.cancel();
 			return;
 		}
@@ -93,10 +93,10 @@ public class FileServer implements IncomingEvent {
 	@Override
 	public void incomingUploadRequest(Downloader downloader) throws IOException {
 		String token = downloader.getToken();
-		LOGGER.info("Incoming filetransfer with token " + token);
+		LOGGER.debug("Incoming filetransfer with token " + token);
 		IncomingDataTransfer upload = uploads.get(token);
 		if (upload == null) {
-			LOGGER.warn("Unknown token " + token);
+			LOGGER.warn("Upload request: Unknown token " + token);
 			downloader.cancel();
 			return;
 		}
diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/maintenance/DeleteOldImages.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/maintenance/DeleteOldImages.java
index 56e7d1d0..23efa2f0 100644
--- a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/maintenance/DeleteOldImages.java
+++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/maintenance/DeleteOldImages.java
@@ -63,6 +63,13 @@ public class DeleteOldImages implements Runnable {
 
 	@Override
 	public void run() {
+		// Get all images currently marked as "Should delete" and reset them to "keep"
+		Set<String> resetList;
+		try {
+			resetList = DbImage.resetDeleteState();
+		} catch (SQLException e1) {
+			resetList = new HashSet<>();
+		}
 		if (!FileSystem.isStorageMounted()) {
 			LOGGER.warn("Will not execute deletion of old images; store seems to be unmounted!");
 			return;
@@ -90,19 +97,22 @@ public class DeleteOldImages implements Runnable {
 			LOGGER.error("Could not mark images to be deleted as invalid. Cleanup of old images failed.");
 			return;
 		}
-		// Delete them permanently only if they expired (at least) one day ago
 		int hardDeleteCount = 0;
 		final long hardDelete = Util.unixTime() - 86400;
 		for (LocalImageVersion version : versions) {
 			if (version.expireTime < hardDelete) {
+				// Delete them permanently only if they expired (at least) one day ago
 				hardDeleteCount++;
 				try {
 					DbImage.setDeletion(DeleteState.SHOULD_DELETE, version.imageVersionId);
 				} catch (SQLException e) {
 				}
 			}
+			// Remove all versions from our reset list that were just disabled again, so we keep those
+			// that have potentially been falsely disabled before
+			resetList.remove(version.imageVersionId);
 		}
-		// Delete base images which no image versions (including invalid ones)
+		// Delete base images with no image versions (including invalid ones)
 		int baseDeleteCount = 0;
 		try {
 			baseDeleteCount = DbImage.deleteOrphanedBases();
@@ -111,6 +121,9 @@ public class DeleteOldImages implements Runnable {
 		}
 		LOGGER.info("Deletion done. Soft: " + (versions.size() - hardDeleteCount) + ", hard: "
 				+ hardDeleteCount + ", base: " + baseDeleteCount);
+		// Aftermath: We might have a list of image versions that have been un-marked from deletion,
+		// and weren't re-marked in this run. This means there might have been clock skew or other problems.
+		// So let's check those images' files, and if they're ok, we also set the 'isvalid' flag again
 	}
 
 	public static StringBuilder hardDeleteImages() {
diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/maintenance/ImageValidCheck.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/maintenance/ImageValidCheck.java
new file mode 100644
index 00000000..0003e0dd
--- /dev/null
+++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/maintenance/ImageValidCheck.java
@@ -0,0 +1,98 @@
+package org.openslx.bwlp.sat.maintenance;
+
+import java.io.File;
+import java.sql.SQLException;
+
+import org.apache.log4j.Logger;
+import org.openslx.bwlp.sat.database.mappers.DbImage;
+import org.openslx.bwlp.sat.database.models.LocalImageVersion;
+import org.openslx.bwlp.sat.util.FileSystem;
+import org.openslx.bwlp.thrift.iface.TNotFoundException;
+import org.openslx.util.Util;
+
+public class ImageValidCheck implements Runnable {
+
+	private static final Logger LOGGER = Logger.getLogger(ImageValidCheck.class);
+
+	private final String versionId;
+
+	public static void check(String versionId) {
+		if (versionId == null)
+			return;
+		Maintenance.trySubmit(new ImageValidCheck(versionId));
+	}
+
+	private ImageValidCheck(String versionId) {
+		this.versionId = versionId;
+	}
+
+	@Override
+	public void run() {
+		if (!FileSystem.waitForStorage()) {
+			LOGGER.warn("Will not check " + versionId + ": Storage not online");
+			return;
+		}
+		LocalImageVersion imageVersion;
+		try {
+			imageVersion = DbImage.getLocalImageData(versionId);
+		} catch (SQLException e) {
+			return;
+		} catch (TNotFoundException e) {
+			LOGGER.warn("Cannot check validity of image version - not found: " + versionId);
+			return;
+		}
+		boolean valid = checkValid(imageVersion);
+		// TODO: We could check the checksums too to be extra safe
+		/*
+		ImageVersionMeta versionDetails;
+		try {
+			versionDetails = DbImage.getVersionDetails(versionId);
+		} catch (TNotFoundException e) {
+			LOGGER.warn("Cannot check validity of image version - not found: " + versionId);
+			return;
+		} catch (SQLException e) {
+			return;
+		}
+		*/
+		if (imageVersion.isValid == valid)
+			return; // nothing changed
+		// Update
+		try {
+			DbImage.markValid(valid, false, imageVersion);
+		} catch (SQLException e) {
+		}
+	}
+
+	private boolean checkValid(LocalImageVersion imageVersion) {
+		if (imageVersion == null)
+			return false;
+		if (imageVersion.expireTime < Util.unixTime()) {
+			LOGGER.info(versionId + ": expired");
+			return false;
+		}
+		if (imageVersion.filePath == null || imageVersion.filePath.isEmpty()) {
+			LOGGER.info(versionId + ": DB does not contain a path");
+			return false;
+		}
+		File path = FileSystem.composeAbsoluteImagePath(imageVersion);
+		if (path == null) {
+			LOGGER.info(versionId + ": path from DB is not valid");
+			return false;
+		}
+		if (!path.exists()) {
+			LOGGER.info(versionId + ": File does not exist (" + path.getAbsolutePath() + ")");
+			return false;
+		}
+		if (!path.canRead()) {
+			LOGGER.info(versionId + ": File exists but not readable (" + path.getAbsolutePath() + ")");
+			return false;
+		}
+		if (path.length() != imageVersion.fileSize) {
+			LOGGER.info(versionId + ": File exists but has wrong size (expected: " + imageVersion.fileSize
+					+ ", found: " + path.length() + ")");
+			return false;
+		}
+		return true;
+	}
+
+}
diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/maintenance/SendExpireWarning.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/maintenance/SendExpireWarning.java
index ba332523..badf97fe 100644
--- a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/maintenance/SendExpireWarning.java
+++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/maintenance/SendExpireWarning.java
@@ -97,7 +97,6 @@ public class SendExpireWarning implements Runnable {
 		final long now = Util.unixTime();
 		for (LocalImageVersion version : versions) {
 			final int days = (int) ((version.expireTime - now) / 86400);
-			LOGGER.debug(version.imageVersionId + " expires in " + days);
 			boolean mailNormal = (version.isValid && (days == 14 || days == 7 || days == 1))
 					|| (!version.isValid && days == 3);
 			boolean mailForced = version.isValid && days == 1;
diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebRpc.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebRpc.java
index 7a7d70ea..a8d769cb 100644
--- a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebRpc.java
+++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebRpc.java
@@ -16,6 +16,7 @@ import org.apache.commons.io.output.ByteArrayOutputStream;
 import org.openslx.bwlp.sat.mail.SmtpMailer;
 import org.openslx.bwlp.sat.mail.SmtpMailer.EncryptionMode;
 import org.openslx.bwlp.sat.maintenance.DeleteOldImages;
+import org.openslx.bwlp.sat.maintenance.ImageValidCheck;
 import org.openslx.util.Util;
 
 import fi.iki.elonen.NanoHTTPD;
@@ -30,9 +31,21 @@ public class WebRpc {
 		if (uri.equals("delete-images")) {
 			return deleteImages();
 		}
+		if (uri.equals("check-image")) {
+			return checkImage(params);
+		}
 		return WebServer.notFound();
 	}
 
+	private static Response checkImage(Map<String, String> params) {
+		String versionId = params.get("versionid");
+		if (versionId == null)
+			return WebServer.badRequest("Missing versionid param");
+		ImageValidCheck.check(versionId);
+		return new NanoHTTPD.Response(NanoHTTPD.Response.Status.OK, "text/plain; charset=utf-8",
+				"Image queued for checking");
+	}
+
 	private static Response deleteImages() {
 		StringBuilder res = DeleteOldImages.hardDeleteImages();
 		if (res == null)
diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebServer.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebServer.java
index 17b28e80..765587e9 100644
--- a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebServer.java
+++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebServer.java
@@ -63,8 +63,6 @@ public class WebServer extends NanoHTTPD {
 				LOGGER.debug("could not parse request body", e);
 				return internalServerError();
 			}
-			LOGGER.debug("Is RPC, passing " + uri.substring(4));
-			LOGGER.debug(session.getParms());
 			return WebRpc.handle(uri.substring(4), session.getParms());
 		}