[server] RPC: Add "scan for orphaned files" function

This function can either just scan, or scan and delete for files on the
vmstore that don't have a matching entry in the database. This can
happen if you restore an older backup after having uploaded new VMs.

References #3321

3 files changed, 117 insertions, 2 deletions

diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbImage.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbImage.java
index ca4c3e3c..6b672c86 100644
--- a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbImage.java
+++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbImage.java
@@ -1067,4 +1067,22 @@ public class DbImage {
 		}
 	}
 
+	/**
+	 * Get all known file names of images, regardless of whether they are working/valid.
+	 */
+	public static Set<String> getAllFilenames() throws SQLException {
+		try (MysqlConnection connection = Database.getConnection()) {
+			MysqlStatement stmt = connection.prepareStatement("SELECT filepath FROM imageversion");
+			ResultSet rs = stmt.executeQuery();
+			Set<String> result = new HashSet<>();
+			while (rs.next()) {
+				result.add(rs.getString("filepath"));
+			}
+			return result;
+		} catch (SQLException e) {
+			LOGGER.error("Query failed in DbImage.getAllFilenames()", e);
+			throw e;
+		}
+	}
+
 }
diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebRpc.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebRpc.java
index 0e47994a..888367fb 100644
--- a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebRpc.java
+++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebRpc.java
@@ -5,16 +5,22 @@ import java.io.IOException;
 import java.io.PrintStream;
 import java.io.PrintWriter;
 import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
 import java.security.InvalidKeyException;
 import java.security.NoSuchAlgorithmException;
 import java.security.spec.InvalidKeySpecException;
+import java.sql.SQLException;
 import java.util.HashMap;
 import java.util.Map;
+import java.util.Set;
 
 import javax.security.auth.login.LoginException;
 
 import org.apache.commons.io.output.ByteArrayOutputStream;
+import org.apache.log4j.Logger;
 import org.openslx.bwlp.sat.database.mappers.DbConfiguration;
+import org.openslx.bwlp.sat.database.mappers.DbImage;
 import org.openslx.bwlp.sat.mail.MailTemplate;
 import org.openslx.bwlp.sat.mail.MailTemplatePlain.Template;
 import org.openslx.bwlp.sat.mail.SmtpMailer;
@@ -23,6 +29,8 @@ import org.openslx.bwlp.sat.maintenance.DeleteOldImages;
 import org.openslx.bwlp.sat.maintenance.ImageValidCheck;
 import org.openslx.bwlp.sat.maintenance.ImageValidCheck.CheckResult;
 import org.openslx.bwlp.sat.maintenance.ImageValidCheck.SubmitResult;
+import org.openslx.bwlp.sat.util.Configuration;
+import org.openslx.bwlp.sat.util.FileSystem;
 import org.openslx.util.Json;
 import org.openslx.util.Util;
 
@@ -31,6 +39,8 @@ import fi.iki.elonen.NanoHTTPD.Response;
 
 public class WebRpc {
 
+	private static final Logger LOGGER = Logger.getLogger(WebRpc.class);
+
 	public static Response handle(String uri, Map<String, String> params) {
 		if (uri.equals("mailtest")) {
 			return mailTest(params);
@@ -47,6 +57,9 @@ public class WebRpc {
 		if (uri.equals("reset-mail-templates")) {
 			return resetMailTemplates();
 		}
+		if (uri.equals("scan-orphaned-files")) {
+			return scanForOrphanedFiles(params);
+		}
 		return WebServer.notFound();
 	}
 
@@ -55,6 +68,85 @@ public class WebRpc {
 		return new NanoHTTPD.Response(NanoHTTPD.Response.Status.OK, "text/plain; charset=utf-8", "OK");
 	}
 
+	/**
+	 * Scan the vmstore for orphaned files and images, return a list.
+	 * If POST param 'action' is 'delete', all those files will be deleted.
+	 */
+	private static Response scanForOrphanedFiles(Map<String, String> params) {
+		if (!FileSystem.isStorageMounted())
+			return WebServer.internalServerError("VMstore not mounted");
+		final Map<String, DeleteResult> orphanedFiles = new HashMap<>();
+		final String baseDir = Configuration.getVmStoreBasePath().toString();
+		final int baseLen = baseDir.length() + (baseDir.endsWith("/") ? 0 : 1);
+		final boolean del = params.containsKey("action") && params.get("action").equals("delete");
+		final Set<String> known; // These we want to keep
+		try {
+			known = DbImage.getAllFilenames();
+		} catch (SQLException e1) {
+			return WebServer.internalServerError("Cannot query list of known images from database");
+		}
+		try {
+			// Consider only regular files, call checkFile for each one
+			Files.find(Configuration.getVmStoreProdPath().toPath(), 8,
+					(filePath, fileAttr) -> fileAttr.isRegularFile())
+					.forEach((fileName) -> checkFile(fileName, orphanedFiles, baseLen, known, del));
+		} catch (IOException e) {
+			return WebServer.internalServerError(e.toString());
+		}
+		return new NanoHTTPD.Response(NanoHTTPD.Response.Status.OK, "application/json; charset=utf-8",
+				Json.serialize(orphanedFiles));
+	}
+
+	private static enum DeleteResult {
+		EXISTS,
+		DELETED,
+		ERROR;
+	}
+
+	/**
+	 * Function called for each file found on the VMstore to determine if it's orphaned.
+	 *
+	 * @param filePath File to check
+	 * @param result Map to add the check result to
+	 * @param baseLen length of the base path we need to strip from the absolute path
+	 * @param known list of known images from the db
+	 * @param doDelete whether to delete all files we found to be orphaned
+	 */
+	private static void checkFile(Path filePath, Map<String, DeleteResult> result, int baseLen,
+			Set<String> known, boolean doDelete) {
+		if (filePath.endsWith("dozmod.lock"))
+			return;
+		final String relativeFileName;
+		try {
+			relativeFileName = filePath.toAbsolutePath().toString().substring(baseLen);
+		} catch (IndexOutOfBoundsException e) {
+			LOGGER.warn("Cannot make image path relative", e);
+			return;
+		}
+		// Handle special dnbd3 files
+		String compareFileName;
+		if (relativeFileName.endsWith(".crc") || relativeFileName.endsWith(".map")) {
+			compareFileName = relativeFileName.substring(0, relativeFileName.length() - 4);
+		} else if (relativeFileName.endsWith(".meta")) {
+			compareFileName = relativeFileName.substring(0, relativeFileName.length() - 5);
+		} else {
+			compareFileName = relativeFileName;
+		}
+		if (!known.contains(compareFileName)) {
+			DeleteResult code = DeleteResult.EXISTS;
+			if (doDelete) {
+				try {
+					Files.delete(filePath);
+					code = DeleteResult.DELETED;
+				} catch (Exception e) {
+					LOGGER.warn("Cannot delete " + filePath, e);
+					code = DeleteResult.ERROR;
+				}
+			}
+			result.put(relativeFileName, code);
+		}
+	}
+
 	private static Response checkImage(Map<String, String> params) {
 		String versionId = params.get("versionid");
 		if (versionId == null)
diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebServer.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebServer.java
index 3e91cfc5..72d8bb95 100644
--- a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebServer.java
+++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebServer.java
@@ -288,10 +288,15 @@ public class WebServer extends NanoHTTPD {
 
 	/**
 	 * Helper for returning "Internal Server Error" Status
+	 * @param body Message 
 	 */
-	public static Response internalServerError() {
+	public static Response internalServerError(String body) {
 		return new NanoHTTPD.Response(NanoHTTPD.Response.Status.INTERNAL_ERROR, "text/plain",
-				"Internal Server Error");
+				body);
+	}
+	
+	public static Response internalServerError() {
+		return internalServerError("Internal Server Error");
 	}
 
 	/**