From 173ea9f81fc576b87dfbe1c0d5997bdb715bea35 Mon Sep 17 00:00:00 2001 From: Simon Rettberg Date: Fri, 25 Sep 2020 16:20:20 +0200 Subject: [server] RPC: Add "scan for orphaned files" function This function can either just scan, or scan and delete for files on the vmstore that don't have a matching entry in the database. This can happen if you restore an older backup after having uploaded new VMs. References #3321 --- .../openslx/bwlp/sat/database/mappers/DbImage.java | 18 +++++ .../main/java/org/openslx/bwlp/sat/web/WebRpc.java | 92 ++++++++++++++++++++++ .../java/org/openslx/bwlp/sat/web/WebServer.java | 9 ++- 3 files changed, 117 insertions(+), 2 deletions(-) diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbImage.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbImage.java index ca4c3e3c..6b672c86 100644 --- a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbImage.java +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbImage.java @@ -1067,4 +1067,22 @@ public class DbImage { } } + /** + * Get all known file names of images, regardless of whether they are working/valid. + */ + public static Set getAllFilenames() throws SQLException { + try (MysqlConnection connection = Database.getConnection()) { + MysqlStatement stmt = connection.prepareStatement("SELECT filepath FROM imageversion"); + ResultSet rs = stmt.executeQuery(); + Set result = new HashSet<>(); + while (rs.next()) { + result.add(rs.getString("filepath")); + } + return result; + } catch (SQLException e) { + LOGGER.error("Query failed in DbImage.getAllFilenames()", e); + throw e; + } + } + } diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebRpc.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebRpc.java index 0e47994a..888367fb 100644 --- a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebRpc.java +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebRpc.java @@ -5,16 +5,22 @@ import java.io.IOException; import java.io.PrintStream; import java.io.PrintWriter; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; import java.security.InvalidKeyException; import java.security.NoSuchAlgorithmException; import java.security.spec.InvalidKeySpecException; +import java.sql.SQLException; import java.util.HashMap; import java.util.Map; +import java.util.Set; import javax.security.auth.login.LoginException; import org.apache.commons.io.output.ByteArrayOutputStream; +import org.apache.log4j.Logger; import org.openslx.bwlp.sat.database.mappers.DbConfiguration; +import org.openslx.bwlp.sat.database.mappers.DbImage; import org.openslx.bwlp.sat.mail.MailTemplate; import org.openslx.bwlp.sat.mail.MailTemplatePlain.Template; import org.openslx.bwlp.sat.mail.SmtpMailer; @@ -23,6 +29,8 @@ import org.openslx.bwlp.sat.maintenance.DeleteOldImages; import org.openslx.bwlp.sat.maintenance.ImageValidCheck; import org.openslx.bwlp.sat.maintenance.ImageValidCheck.CheckResult; import org.openslx.bwlp.sat.maintenance.ImageValidCheck.SubmitResult; +import org.openslx.bwlp.sat.util.Configuration; +import org.openslx.bwlp.sat.util.FileSystem; import org.openslx.util.Json; import org.openslx.util.Util; @@ -31,6 +39,8 @@ import fi.iki.elonen.NanoHTTPD.Response; public class WebRpc { + private static final Logger LOGGER = Logger.getLogger(WebRpc.class); + public static Response handle(String uri, Map params) { if (uri.equals("mailtest")) { return mailTest(params); @@ -47,6 +57,9 @@ public class WebRpc { if (uri.equals("reset-mail-templates")) { return resetMailTemplates(); } + if (uri.equals("scan-orphaned-files")) { + return scanForOrphanedFiles(params); + } return WebServer.notFound(); } @@ -55,6 +68,85 @@ public class WebRpc { return new NanoHTTPD.Response(NanoHTTPD.Response.Status.OK, "text/plain; charset=utf-8", "OK"); } + /** + * Scan the vmstore for orphaned files and images, return a list. + * If POST param 'action' is 'delete', all those files will be deleted. + */ + private static Response scanForOrphanedFiles(Map params) { + if (!FileSystem.isStorageMounted()) + return WebServer.internalServerError("VMstore not mounted"); + final Map orphanedFiles = new HashMap<>(); + final String baseDir = Configuration.getVmStoreBasePath().toString(); + final int baseLen = baseDir.length() + (baseDir.endsWith("/") ? 0 : 1); + final boolean del = params.containsKey("action") && params.get("action").equals("delete"); + final Set known; // These we want to keep + try { + known = DbImage.getAllFilenames(); + } catch (SQLException e1) { + return WebServer.internalServerError("Cannot query list of known images from database"); + } + try { + // Consider only regular files, call checkFile for each one + Files.find(Configuration.getVmStoreProdPath().toPath(), 8, + (filePath, fileAttr) -> fileAttr.isRegularFile()) + .forEach((fileName) -> checkFile(fileName, orphanedFiles, baseLen, known, del)); + } catch (IOException e) { + return WebServer.internalServerError(e.toString()); + } + return new NanoHTTPD.Response(NanoHTTPD.Response.Status.OK, "application/json; charset=utf-8", + Json.serialize(orphanedFiles)); + } + + private static enum DeleteResult { + EXISTS, + DELETED, + ERROR; + } + + /** + * Function called for each file found on the VMstore to determine if it's orphaned. + * + * @param filePath File to check + * @param result Map to add the check result to + * @param baseLen length of the base path we need to strip from the absolute path + * @param known list of known images from the db + * @param doDelete whether to delete all files we found to be orphaned + */ + private static void checkFile(Path filePath, Map result, int baseLen, + Set known, boolean doDelete) { + if (filePath.endsWith("dozmod.lock")) + return; + final String relativeFileName; + try { + relativeFileName = filePath.toAbsolutePath().toString().substring(baseLen); + } catch (IndexOutOfBoundsException e) { + LOGGER.warn("Cannot make image path relative", e); + return; + } + // Handle special dnbd3 files + String compareFileName; + if (relativeFileName.endsWith(".crc") || relativeFileName.endsWith(".map")) { + compareFileName = relativeFileName.substring(0, relativeFileName.length() - 4); + } else if (relativeFileName.endsWith(".meta")) { + compareFileName = relativeFileName.substring(0, relativeFileName.length() - 5); + } else { + compareFileName = relativeFileName; + } + if (!known.contains(compareFileName)) { + DeleteResult code = DeleteResult.EXISTS; + if (doDelete) { + try { + Files.delete(filePath); + code = DeleteResult.DELETED; + } catch (Exception e) { + LOGGER.warn("Cannot delete " + filePath, e); + code = DeleteResult.ERROR; + } + } + result.put(relativeFileName, code); + } + } + private static Response checkImage(Map params) { String versionId = params.get("versionid"); if (versionId == null) diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebServer.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebServer.java index 3e91cfc5..72d8bb95 100644 --- a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebServer.java +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebServer.java @@ -288,10 +288,15 @@ public class WebServer extends NanoHTTPD { /** * Helper for returning "Internal Server Error" Status + * @param body Message */ - public static Response internalServerError() { + public static Response internalServerError(String body) { return new NanoHTTPD.Response(NanoHTTPD.Response.Status.INTERNAL_ERROR, "text/plain", - "Internal Server Error"); + body); + } + + public static Response internalServerError() { + return internalServerError("Internal Server Error"); } /** -- cgit v1.2.3-55-g7522