From 68eab959ffd46ee39130c713643d11c1ff26467e Mon Sep 17 00:00:00 2001 From: Simon Rettberg Date: Fri, 25 Sep 2020 16:19:01 +0200 Subject: [server] switch to Java 1.8 --- dozentenmodulserver/pom.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'dozentenmodulserver') diff --git a/dozentenmodulserver/pom.xml b/dozentenmodulserver/pom.xml index f660d51d..ac41c772 100644 --- a/dozentenmodulserver/pom.xml +++ b/dozentenmodulserver/pom.xml @@ -38,8 +38,8 @@ maven-compiler-plugin 3.6.1 - 1.7 - 1.7 + 1.8 + 1.8 -- cgit v1.2.3-55-g7522 From 173ea9f81fc576b87dfbe1c0d5997bdb715bea35 Mon Sep 17 00:00:00 2001 From: Simon Rettberg Date: Fri, 25 Sep 2020 16:20:20 +0200 Subject: [server] RPC: Add "scan for orphaned files" function This function can either just scan, or scan and delete for files on the vmstore that don't have a matching entry in the database. This can happen if you restore an older backup after having uploaded new VMs. References #3321 --- .../openslx/bwlp/sat/database/mappers/DbImage.java | 18 +++++ .../main/java/org/openslx/bwlp/sat/web/WebRpc.java | 92 ++++++++++++++++++++++ .../java/org/openslx/bwlp/sat/web/WebServer.java | 9 ++- 3 files changed, 117 insertions(+), 2 deletions(-) (limited to 'dozentenmodulserver') diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbImage.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbImage.java index ca4c3e3c..6b672c86 100644 --- a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbImage.java +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbImage.java @@ -1067,4 +1067,22 @@ public class DbImage { } } + /** + * Get all known file names of images, regardless of whether they are working/valid. + */ + public static Set getAllFilenames() throws SQLException { + try (MysqlConnection connection = Database.getConnection()) { + MysqlStatement stmt = connection.prepareStatement("SELECT filepath FROM imageversion"); + ResultSet rs = stmt.executeQuery(); + Set result = new HashSet<>(); + while (rs.next()) { + result.add(rs.getString("filepath")); + } + return result; + } catch (SQLException e) { + LOGGER.error("Query failed in DbImage.getAllFilenames()", e); + throw e; + } + } + } diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebRpc.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebRpc.java index 0e47994a..888367fb 100644 --- a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebRpc.java +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebRpc.java @@ -5,16 +5,22 @@ import java.io.IOException; import java.io.PrintStream; import java.io.PrintWriter; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; import java.security.InvalidKeyException; import java.security.NoSuchAlgorithmException; import java.security.spec.InvalidKeySpecException; +import java.sql.SQLException; import java.util.HashMap; import java.util.Map; +import java.util.Set; import javax.security.auth.login.LoginException; import org.apache.commons.io.output.ByteArrayOutputStream; +import org.apache.log4j.Logger; import org.openslx.bwlp.sat.database.mappers.DbConfiguration; +import org.openslx.bwlp.sat.database.mappers.DbImage; import org.openslx.bwlp.sat.mail.MailTemplate; import org.openslx.bwlp.sat.mail.MailTemplatePlain.Template; import org.openslx.bwlp.sat.mail.SmtpMailer; @@ -23,6 +29,8 @@ import org.openslx.bwlp.sat.maintenance.DeleteOldImages; import org.openslx.bwlp.sat.maintenance.ImageValidCheck; import org.openslx.bwlp.sat.maintenance.ImageValidCheck.CheckResult; import org.openslx.bwlp.sat.maintenance.ImageValidCheck.SubmitResult; +import org.openslx.bwlp.sat.util.Configuration; +import org.openslx.bwlp.sat.util.FileSystem; import org.openslx.util.Json; import org.openslx.util.Util; @@ -31,6 +39,8 @@ import fi.iki.elonen.NanoHTTPD.Response; public class WebRpc { + private static final Logger LOGGER = Logger.getLogger(WebRpc.class); + public static Response handle(String uri, Map params) { if (uri.equals("mailtest")) { return mailTest(params); @@ -47,6 +57,9 @@ public class WebRpc { if (uri.equals("reset-mail-templates")) { return resetMailTemplates(); } + if (uri.equals("scan-orphaned-files")) { + return scanForOrphanedFiles(params); + } return WebServer.notFound(); } @@ -55,6 +68,85 @@ public class WebRpc { return new NanoHTTPD.Response(NanoHTTPD.Response.Status.OK, "text/plain; charset=utf-8", "OK"); } + /** + * Scan the vmstore for orphaned files and images, return a list. + * If POST param 'action' is 'delete', all those files will be deleted. + */ + private static Response scanForOrphanedFiles(Map params) { + if (!FileSystem.isStorageMounted()) + return WebServer.internalServerError("VMstore not mounted"); + final Map orphanedFiles = new HashMap<>(); + final String baseDir = Configuration.getVmStoreBasePath().toString(); + final int baseLen = baseDir.length() + (baseDir.endsWith("/") ? 0 : 1); + final boolean del = params.containsKey("action") && params.get("action").equals("delete"); + final Set known; // These we want to keep + try { + known = DbImage.getAllFilenames(); + } catch (SQLException e1) { + return WebServer.internalServerError("Cannot query list of known images from database"); + } + try { + // Consider only regular files, call checkFile for each one + Files.find(Configuration.getVmStoreProdPath().toPath(), 8, + (filePath, fileAttr) -> fileAttr.isRegularFile()) + .forEach((fileName) -> checkFile(fileName, orphanedFiles, baseLen, known, del)); + } catch (IOException e) { + return WebServer.internalServerError(e.toString()); + } + return new NanoHTTPD.Response(NanoHTTPD.Response.Status.OK, "application/json; charset=utf-8", + Json.serialize(orphanedFiles)); + } + + private static enum DeleteResult { + EXISTS, + DELETED, + ERROR; + } + + /** + * Function called for each file found on the VMstore to determine if it's orphaned. + * + * @param filePath File to check + * @param result Map to add the check result to + * @param baseLen length of the base path we need to strip from the absolute path + * @param known list of known images from the db + * @param doDelete whether to delete all files we found to be orphaned + */ + private static void checkFile(Path filePath, Map result, int baseLen, + Set known, boolean doDelete) { + if (filePath.endsWith("dozmod.lock")) + return; + final String relativeFileName; + try { + relativeFileName = filePath.toAbsolutePath().toString().substring(baseLen); + } catch (IndexOutOfBoundsException e) { + LOGGER.warn("Cannot make image path relative", e); + return; + } + // Handle special dnbd3 files + String compareFileName; + if (relativeFileName.endsWith(".crc") || relativeFileName.endsWith(".map")) { + compareFileName = relativeFileName.substring(0, relativeFileName.length() - 4); + } else if (relativeFileName.endsWith(".meta")) { + compareFileName = relativeFileName.substring(0, relativeFileName.length() - 5); + } else { + compareFileName = relativeFileName; + } + if (!known.contains(compareFileName)) { + DeleteResult code = DeleteResult.EXISTS; + if (doDelete) { + try { + Files.delete(filePath); + code = DeleteResult.DELETED; + } catch (Exception e) { + LOGGER.warn("Cannot delete " + filePath, e); + code = DeleteResult.ERROR; + } + } + result.put(relativeFileName, code); + } + } + private static Response checkImage(Map params) { String versionId = params.get("versionid"); if (versionId == null) diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebServer.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebServer.java index 3e91cfc5..72d8bb95 100644 --- a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebServer.java +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebServer.java @@ -288,10 +288,15 @@ public class WebServer extends NanoHTTPD { /** * Helper for returning "Internal Server Error" Status + * @param body Message */ - public static Response internalServerError() { + public static Response internalServerError(String body) { return new NanoHTTPD.Response(NanoHTTPD.Response.Status.INTERNAL_ERROR, "text/plain", - "Internal Server Error"); + body); + } + + public static Response internalServerError() { + return internalServerError("Internal Server Error"); } /** -- cgit v1.2.3-55-g7522 From 1e813cfe03f5035f3c9b842090a5377f6c9d4a05 Mon Sep 17 00:00:00 2001 From: Simon Rettberg Date: Fri, 25 Sep 2020 16:23:01 +0200 Subject: [server] Formatting --- .../src/main/java/org/openslx/bwlp/sat/web/WebRpc.java | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) (limited to 'dozentenmodulserver') diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebRpc.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebRpc.java index 888367fb..2709e567 100644 --- a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebRpc.java +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebRpc.java @@ -180,7 +180,8 @@ public class WebRpc { StringBuilder res = DeleteOldImages.hardDeleteImages(); if (res == null) return WebServer.internalServerError(); - return new NanoHTTPD.Response(NanoHTTPD.Response.Status.OK, "text/plain; charset=utf-8", res.toString()); + return new NanoHTTPD.Response(NanoHTTPD.Response.Status.OK, "text/plain; charset=utf-8", + res.toString()); } /** @@ -224,11 +225,9 @@ public class WebRpc { } smtpc = null; } - boolean ret = false; - if (smtpc != null) { - + if (smtpc != null) { MailTemplate template = DbConfiguration.getMailTemplate(Template.TEST_MAIL); Map templateArgs = new HashMap<>(); templateArgs.put("host", host); @@ -241,8 +240,8 @@ public class WebRpc { ret = smtpc.send(recipient, "bwLehrpool Mail Test", msg, ""); } try { - baos.write(("\n\n-----------------------------------------\nTestergebnis: " + (ret ? "" : "nicht ") - + "erfolgreich").getBytes(StandardCharsets.UTF_8)); + baos.write(("\n\n-----------------------------------------\nTestergebnis: " + + (ret ? "" : "nicht ") + "erfolgreich").getBytes(StandardCharsets.UTF_8)); } catch (IOException e) { } return new NanoHTTPD.Response(NanoHTTPD.Response.Status.OK, "text/plain; charset=utf-8", -- cgit v1.2.3-55-g7522 From 8a7a51538018019d7122dce27c4ec8ba6cfe93e6 Mon Sep 17 00:00:00 2001 From: Simon Rettberg Date: Fri, 25 Sep 2020 16:23:23 +0200 Subject: [server] Remove unused exception --- .../src/main/java/org/openslx/bwlp/sat/database/mappers/DbLecture.java | 3 +-- .../src/main/java/org/openslx/bwlp/sat/web/WebServer.java | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) (limited to 'dozentenmodulserver') diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbLecture.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbLecture.java index a57be8da..3dc58d47 100644 --- a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbLecture.java +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbLecture.java @@ -30,7 +30,6 @@ import org.openslx.bwlp.thrift.iface.TNotFoundException; import org.openslx.bwlp.thrift.iface.UserInfo; import org.openslx.util.Json; import org.openslx.util.Util; -import org.openslx.util.vm.UnsupportedVirtualizerFormatException; import org.openslx.util.vm.VmMetaData; import org.openslx.util.vm.VmMetaData.UsbSpeed; @@ -552,7 +551,7 @@ public class DbLecture { } public static LaunchData getClientLaunchData(String lectureId) throws SQLException, - TNotFoundException, UnsupportedVirtualizerFormatException { + TNotFoundException { LaunchData retval = new LaunchData(); byte[] config; String lectureName; diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebServer.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebServer.java index 72d8bb95..6357e411 100644 --- a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebServer.java +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebServer.java @@ -31,7 +31,6 @@ import org.openslx.bwlp.thrift.iface.TNotFoundException; import org.openslx.util.GrowingThreadPoolExecutor; import org.openslx.util.Json; import org.openslx.util.Util; -import org.openslx.util.vm.UnsupportedVirtualizerFormatException; import org.simpleframework.xml.Serializer; import org.simpleframework.xml.core.Persister; @@ -167,7 +166,7 @@ public class WebServer extends NanoHTTPD { final LaunchData ld; try { ld = DbLecture.getClientLaunchData(lectureId); - } catch (UnsupportedVirtualizerFormatException | TNotFoundException e) { + } catch (TNotFoundException e) { // TODO better virt error handling return notFound(); } catch (SQLException e) { -- cgit v1.2.3-55-g7522 From 5e143d493eb088e3f5dce86e8906483e7efc909d Mon Sep 17 00:00:00 2001 From: Simon Rettberg Date: Mon, 28 Sep 2020 12:02:54 +0200 Subject: [server] Abort orphan scan/delete if known image list is empty On the off-chance that something goes wron when querying, or the database got emptied, don't just blindly wipe the whole vmstore. It's probably smarter to restore a backup first, or if you really mean to start afresh, just empty the vmstore manually, or just upload one VM and then use this feature again. --- dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebRpc.java | 3 +++ 1 file changed, 3 insertions(+) (limited to 'dozentenmodulserver') diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebRpc.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebRpc.java index 2709e567..1d5b27e7 100644 --- a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebRpc.java +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebRpc.java @@ -85,6 +85,9 @@ public class WebRpc { } catch (SQLException e1) { return WebServer.internalServerError("Cannot query list of known images from database"); } + if (known.isEmpty()) { + return WebServer.internalServerError("SAFTY CHECK: Known image list empty, aborting"); + } try { // Consider only regular files, call checkFile for each one Files.find(Configuration.getVmStoreProdPath().toPath(), 8, -- cgit v1.2.3-55-g7522 From fe9c0cc3445485221b12118c2f5e34cccde82105 Mon Sep 17 00:00:00 2001 From: Simon Rettberg Date: Mon, 28 Sep 2020 12:24:58 +0200 Subject: [server] Up the safety game even more: Don't delete if disjoint If non of the files in the known image list matches the list of files we find on the vmstore, don't delete anything. It's possible we mounted the wrong vmstore (test server vs. production system) and would totally ruin the day for someone. --- .../main/java/org/openslx/bwlp/sat/web/WebRpc.java | 53 ++++++++++++++-------- 1 file changed, 35 insertions(+), 18 deletions(-) (limited to 'dozentenmodulserver') diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebRpc.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebRpc.java index 1d5b27e7..3a6f39ad 100644 --- a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebRpc.java +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebRpc.java @@ -7,13 +7,16 @@ import java.io.PrintWriter; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.Paths; import java.security.InvalidKeyException; import java.security.NoSuchAlgorithmException; import java.security.spec.InvalidKeySpecException; import java.sql.SQLException; import java.util.HashMap; import java.util.Map; +import java.util.Map.Entry; import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; import javax.security.auth.login.LoginException; @@ -88,14 +91,33 @@ public class WebRpc { if (known.isEmpty()) { return WebServer.internalServerError("SAFTY CHECK: Known image list empty, aborting"); } + AtomicInteger matches = new AtomicInteger(); try { // Consider only regular files, call checkFile for each one Files.find(Configuration.getVmStoreProdPath().toPath(), 8, (filePath, fileAttr) -> fileAttr.isRegularFile()) - .forEach((fileName) -> checkFile(fileName, orphanedFiles, baseLen, known, del)); + .forEach((fileName) -> checkFile(fileName, orphanedFiles, baseLen, known, matches)); } catch (IOException e) { return WebServer.internalServerError(e.toString()); } + if (del) { + for (Entry it : orphanedFiles.entrySet()) { + if (matches.get() == 0) { + // Don't delete anything if the set of known files and the set of files we actually + // found are disjoint + it.setValue(DeleteResult.SAFETY_ABORT); + continue; + } + Path filePath = Paths.get(baseDir + "/" + it.getKey()); + try { + Files.delete(filePath); + it.setValue(DeleteResult.DELETED); + } catch (Exception e) { + LOGGER.warn("Cannot delete " + filePath, e); + it.setValue(DeleteResult.ERROR); + } + } + } return new NanoHTTPD.Response(NanoHTTPD.Response.Status.OK, "application/json; charset=utf-8", Json.serialize(orphanedFiles)); } @@ -103,20 +125,23 @@ public class WebRpc { private static enum DeleteResult { EXISTS, DELETED, - ERROR; + ERROR, + SAFETY_ABORT; } /** - * Function called for each file found on the VMstore to determine if it's orphaned. + * Function called for each file found on the VMstore to determine if it's + * orphaned. * * @param filePath File to check * @param result Map to add the check result to - * @param baseLen length of the base path we need to strip from the absolute path + * @param baseLen length of the base path we need to strip from the absolute + * path * @param known list of known images from the db - * @param doDelete whether to delete all files we found to be orphaned + * @param matches counter for files that match a DB entry */ private static void checkFile(Path filePath, Map result, int baseLen, - Set known, boolean doDelete) { + Set known, AtomicInteger matches) { if (filePath.endsWith("dozmod.lock")) return; final String relativeFileName; @@ -135,18 +160,10 @@ public class WebRpc { } else { compareFileName = relativeFileName; } - if (!known.contains(compareFileName)) { - DeleteResult code = DeleteResult.EXISTS; - if (doDelete) { - try { - Files.delete(filePath); - code = DeleteResult.DELETED; - } catch (Exception e) { - LOGGER.warn("Cannot delete " + filePath, e); - code = DeleteResult.ERROR; - } - } - result.put(relativeFileName, code); + if (known.contains(compareFileName)) { + matches.incrementAndGet(); + } else { + result.put(relativeFileName, DeleteResult.EXISTS); } } -- cgit v1.2.3-55-g7522