diff options
9 files changed, 198 insertions, 20 deletions
diff --git a/dozentenmodulserver/setup/sat-01-schema.sql b/dozentenmodulserver/setup/sat-01-schema.sql index d44b0ed2..c6caef39 100644 --- a/dozentenmodulserver/setup/sat-01-schema.sql +++ b/dozentenmodulserver/setup/sat-01-schema.sql @@ -75,6 +75,7 @@ CREATE TABLE IF NOT EXISTS `imageversion` ( `isprocessed` tinyint(1) NOT NULL, `mastersha1` binary(20) DEFAULT NULL, `virtualizerconfig` blob NULL DEFAULT NULL COMMENT 'Specific configuration of the virtualizer for this image. For vmware, this is basically a dump of the *.vmx.', + `deletestate` ENUM( 'KEEP', 'SHOULD_DELETE', 'WANT_DELETE' ) CHARACTER SET ascii COLLATE ascii_bin NOT NULL DEFAULT 'KEEP' PRIMARY KEY (`imageversionid`), KEY `version_access` (`imagebaseid`,`isvalid`,`createtime`), KEY `fk_imageversion_2_idx` (`uploaderid`), diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbImage.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbImage.java index 40d97657..2bf26208 100644 --- a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbImage.java +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbImage.java @@ -115,14 +115,15 @@ public class DbImage { } } - private final static String localImageBaseSql = "SELECT" - + " v.imageversionid, v.imagebaseid, v.filepath, v.filesize, v.uploaderid, v.createtime, v.expiretime, v.isvalid" + private final static String localImageBaseSql = "SELECT v.imageversionid, v.imagebaseid," + + " v.filepath, v.filesize, v.uploaderid, v.createtime, v.expiretime, v.isvalid, v.deletestate" + " FROM imageversion v"; private static LocalImageVersion toLocalImageVersion(ResultSet rs) throws SQLException { return new LocalImageVersion(rs.getString("imageversionid"), rs.getString("imagebaseid"), rs.getString("filepath"), rs.getLong("filesize"), rs.getString("uploaderid"), - rs.getLong("createtime"), rs.getLong("expiretime"), rs.getBoolean("isvalid")); + rs.getLong("createtime"), rs.getLong("expiretime"), rs.getBoolean("isvalid"), + rs.getString("deletestate")); } public static LocalImageVersion getLocalImageData(String imageVersionId) throws TNotFoundException, @@ -491,6 +492,8 @@ public class DbImage { + " expiretime = 1234567890, isvalid = 0" + " WHERE imageversionid = :versionid"); affectedList = new ArrayList<>(imageVersionIds.length); for (String imageVersionId : imageVersionIds) { + if (imageVersionId == null) + continue; disableStmt.setString("versionid", imageVersionId); if (disableStmt.executeUpdate() != 0) { affectedList.add(imageVersionId); @@ -547,7 +550,7 @@ public class DbImage { verStmt.executeUpdate(); writeChunks(connection, imageVersionId, chunks); LocalImageVersion liv = new LocalImageVersion(imageVersionId, imageBaseId, filePath, fileSize, - owner.userId, nowSecs, expireTime, true); + owner.userId, nowSecs, expireTime, true, DeleteState.KEEP.toString()); DbLecture.autoUpdateUsedImage(connection, imageBaseId, liv); // Update edit timestamp and edit user MysqlStatement baseStmt = connection.prepareStatement("UPDATE imagebase SET" @@ -633,8 +636,9 @@ public class DbImage { } } - public static void deletePermanently(LocalImageVersion image) throws SQLException { + public static void deleteVersionPermanently(LocalImageVersion image) throws SQLException { try (MysqlConnection connection = Database.getConnection()) { + DbLecture.unlinkFromImageVersion(connection, image.imageVersionId); DbLecture.deletePermanently(connection, image); MysqlStatement stmt = connection.prepareStatement("DELETE FROM imageversion" + " WHERE imageversionid = :imageversionid"); @@ -721,7 +725,8 @@ public class DbImage { if (version.imageVersionId.equals(changingImageVersionId)) { changingVersion = version; } - if (version.isValid && (latestVersion == null || version.createTime > latestVersion.createTime)) { + if (version.deleteState == DeleteState.KEEP && version.isValid + && (latestVersion == null || version.createTime > latestVersion.createTime)) { File versionFile = FileSystem.composeAbsoluteImagePath(version); if (versionFile != null) { if (versionFile.canRead() && versionFile.length() == version.fileSize) { @@ -823,9 +828,10 @@ public class DbImage { ResultSet rs = stmt.executeQuery(); List<LocalImageVersion> list = new ArrayList<>(); while (rs.next()) { + // Copy of helper so we can pass 0 for expire date list.add(new LocalImageVersion(rs.getString("imageversionid"), rs.getString("imagebaseid"), rs.getString("filepath"), rs.getLong("filesize"), rs.getString("uploaderid"), - rs.getLong("createtime"), 0, rs.getBoolean("isvalid"))); + rs.getLong("createtime"), 0, rs.getBoolean("isvalid"), rs.getString("deletestate"))); } return list; } catch (SQLException e) { @@ -895,4 +901,67 @@ public class DbImage { return list; } + public enum DeleteState { + KEEP, + SHOULD_DELETE, + DO_DELETE; + } + + public static void setDeletion(DeleteState shouldDelete, String... imageVersionIds) throws SQLException { + if (imageVersionIds == null || imageVersionIds.length == 0 || shouldDelete == null) + return; + String ignoredOldState; + if (shouldDelete == DeleteState.SHOULD_DELETE) { + ignoredOldState = DeleteState.DO_DELETE.toString(); + } else { + ignoredOldState = "invalid"; + } + try (MysqlConnection connection = Database.getConnection()) { + MysqlStatement stmt = connection.prepareStatement("UPDATE imageversion SET deletestate = :newstate" + + " WHERE imageversionid = :imageversionid AND deletestate <> :oldstate"); + stmt.setString("newstate", shouldDelete.toString()); + stmt.setString("oldstate", ignoredOldState); + for (String imageVersionId : imageVersionIds) { + if (imageVersionId == null) + continue; + stmt.setString("imageversionid", imageVersionId); + stmt.executeUpdate(); + } + connection.commit(); + } catch (SQLException e) { + LOGGER.error("Query failed in DbImage.setDeletion()", e); + throw e; + } + } + + public static List<LocalImageVersion> getLocalWithState(DeleteState state) throws SQLException { + try (MysqlConnection connection = Database.getConnection()) { + MysqlStatement stmt = connection.prepareStatement(localImageBaseSql + + " WHERE deletestate = :deletestate"); + stmt.setString("deletestate", state.toString()); + ResultSet rs = stmt.executeQuery(); + List<LocalImageVersion> list = new ArrayList<>(); + while (rs.next()) { + list.add(toLocalImageVersion(rs)); + } + return list; + } catch (SQLException e) { + LOGGER.error("Query failed in DbImage.getLocalWithState()", e); + throw e; + } + } + + public static void deleteBasePermanently(String imageBaseId) throws SQLException { + try (MysqlConnection connection = Database.getConnection()) { + MysqlStatement stmt = connection.prepareStatement("DELETE FROM imagebase" + + " WHERE imagebaseid = :imagebaseid"); + stmt.setString("imagebaseid", imageBaseId); + stmt.executeUpdate(); + connection.commit(); + } catch (SQLException e) { + LOGGER.error("Query failed in DbImage.deleteBasePermanently()", e); + throw e; + } + } + } diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbLecture.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbLecture.java index b93e1464..0e6b4ec1 100644 --- a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbLecture.java +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbLecture.java @@ -511,4 +511,21 @@ public class DbLecture { } } + public static void unlinkFromImageVersion(String imageVersionId) throws SQLException { + try (MysqlConnection connection = Database.getConnection()) { + unlinkFromImageVersion(connection, imageVersionId); + connection.commit(); + } catch (SQLException e) { + LOGGER.error("Query failed in DbLecture.unlinkFromImageVersion()", e); + throw e; + } + } + + protected static void unlinkFromImageVersion(MysqlConnection connection, String imageVersionId) + throws SQLException { + MysqlStatement stmt = connection.prepareStatement("UPDATE lecture SET imageversionid = NULL" + + " WHERE imageversionid = :imageversionid"); + stmt.setString("imageversionid", imageVersionId); + } + } diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/models/LocalImageVersion.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/models/LocalImageVersion.java index 96b6e44d..4bce4b37 100644 --- a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/models/LocalImageVersion.java +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/models/LocalImageVersion.java @@ -1,5 +1,7 @@ package org.openslx.bwlp.sat.database.models; +import org.openslx.bwlp.sat.database.mappers.DbImage.DeleteState; + public class LocalImageVersion { public final String imageVersionId; @@ -18,8 +20,10 @@ public class LocalImageVersion { public final long expireTime; + public final DeleteState deleteState; + public LocalImageVersion(String imageVersionId, String imageBaseId, String filePath, long fileSize, - String uploaderId, long createTime, long expireTime, boolean isValid) { + String uploaderId, long createTime, long expireTime, boolean isValid, String deleteState) { this.imageVersionId = imageVersionId; this.imageBaseId = imageBaseId; this.filePath = filePath; @@ -28,6 +32,13 @@ public class LocalImageVersion { this.createTime = createTime; this.expireTime = expireTime; this.isValid = isValid; + DeleteState ds; + try { + ds = DeleteState.valueOf(deleteState); + } catch (Exception e) { + ds = DeleteState.KEEP; + } + this.deleteState = ds; } @Override diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/fileserv/IncomingDataTransfer.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/fileserv/IncomingDataTransfer.java index d008f5db..fa8647d9 100644 --- a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/fileserv/IncomingDataTransfer.java +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/fileserv/IncomingDataTransfer.java @@ -37,6 +37,8 @@ import org.openslx.filetransfer.util.FileChunk; import org.openslx.filetransfer.util.HashChecker; import org.openslx.filetransfer.util.HashChecker.HashCheckCallback; import org.openslx.filetransfer.util.HashChecker.HashResult; +import org.openslx.util.vm.DiskImage; +import org.openslx.util.vm.DiskImage.UnknownImageFormatException; public class IncomingDataTransfer extends AbstractTransfer implements HashCheckCallback { @@ -311,7 +313,12 @@ public class IncomingDataTransfer extends AbstractTransfer implements HashCheckC return; LOGGER.info("Finalizing uploaded image " + image.imageName); // Ready to go. First step: Rename temp file to something usable - File destination = new File(tmpFileName.getParent(), Formatter.vmName(owner, image.imageName)); + String ext = "img"; + try { + ext = new DiskImage(tmpFileName).format.extension; + } catch (IOException | UnknownImageFormatException e1) { + } + File destination = new File(tmpFileName.getParent(), Formatter.vmName(owner, image.imageName, ext)); // Sanity check: destination should be a sub directory of the vmStorePath String relPath = FileSystem.getRelativePath(destination, Configuration.getVmStoreBasePath()); if (relPath == null) { diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/maintenance/DeleteOldImages.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/maintenance/DeleteOldImages.java index 531f169a..56e7d1d0 100644 --- a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/maintenance/DeleteOldImages.java +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/maintenance/DeleteOldImages.java @@ -9,10 +9,13 @@ import java.util.concurrent.TimeUnit; import org.apache.log4j.Logger; import org.joda.time.DateTime; import org.openslx.bwlp.sat.database.mappers.DbImage; +import org.openslx.bwlp.sat.database.mappers.DbImage.DeleteState; +import org.openslx.bwlp.sat.database.mappers.DbLecture; import org.openslx.bwlp.sat.database.models.LocalImageVersion; import org.openslx.bwlp.sat.util.FileSystem; import org.openslx.util.QuickTimer; import org.openslx.util.QuickTimer.Task; +import org.openslx.util.Util; /** * Delete old image versions (images that reached their expire time). @@ -46,7 +49,7 @@ public class DeleteOldImages implements Runnable { } }, TimeUnit.MINUTES.toMillis(5), TimeUnit.MINUTES.toMillis(5)); } - + public synchronized static void start() { if (blockedUntil > System.currentTimeMillis()) return; @@ -89,15 +92,13 @@ public class DeleteOldImages implements Runnable { } // Delete them permanently only if they expired (at least) one day ago int hardDeleteCount = 0; - final long hardDelete = (System.currentTimeMillis() / 1000) - 86400; + final long hardDelete = Util.unixTime() - 86400; for (LocalImageVersion version : versions) { if (version.expireTime < hardDelete) { hardDeleteCount++; - FileSystem.deleteImageRelatedFiles(version); try { - DbImage.deletePermanently(version); + DbImage.setDeletion(DeleteState.SHOULD_DELETE, version.imageVersionId); } catch (SQLException e) { - // Logging done in method } } } @@ -112,4 +113,46 @@ public class DeleteOldImages implements Runnable { + hardDeleteCount + ", base: " + baseDeleteCount); } + public static StringBuilder hardDeleteImages() { + StringBuilder sb = new StringBuilder(); + List<LocalImageVersion> deletables; + try { + deletables = DbImage.getLocalWithState(DeleteState.DO_DELETE); + } catch (SQLException e2) { + return null; + } + for (LocalImageVersion version : deletables) { + FileSystem.deleteImageRelatedFiles(version); + try { + DbLecture.unlinkFromImageVersion(version.imageVersionId); + DbImage.deleteVersionPermanently(version); + } catch (SQLException e) { + writeln(sb, version.imageVersionId, ": Cannot delete image: ", e.getMessage()); + } + writeln(sb, version.imageVersionId, ": OK"); + } + writeln(sb, "Done"); + return sb; + } + + private static void writeln(StringBuilder sb, String... parts) { + for (String s : parts) { + if (s == null) { + sb.append("(null)"); + } else { + sb.append(s); + } + } + sb.append('\n'); + } + + public static void hardDeleteImagesAsync() { + Maintenance.trySubmit(new Runnable() { + @Override + public void run() { + hardDeleteImages(); + } + }); + } + } diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/thrift/ServerHandler.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/thrift/ServerHandler.java index 2fe65d86..751e480e 100644 --- a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/thrift/ServerHandler.java +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/thrift/ServerHandler.java @@ -9,6 +9,7 @@ import org.apache.log4j.Logger; import org.apache.thrift.TException; import org.openslx.bwlp.sat.RuntimeConfig; import org.openslx.bwlp.sat.database.mappers.DbImage; +import org.openslx.bwlp.sat.database.mappers.DbImage.DeleteState; import org.openslx.bwlp.sat.database.mappers.DbImagePermissions; import org.openslx.bwlp.sat.database.mappers.DbLecture; import org.openslx.bwlp.sat.database.mappers.DbLecturePermissions; @@ -19,6 +20,7 @@ import org.openslx.bwlp.sat.fileserv.FileServer; import org.openslx.bwlp.sat.fileserv.IncomingDataTransfer; import org.openslx.bwlp.sat.fileserv.OutgoingDataTransfer; import org.openslx.bwlp.sat.fileserv.SyncTransferHandler; +import org.openslx.bwlp.sat.maintenance.DeleteOldImages; import org.openslx.bwlp.sat.permissions.User; import org.openslx.bwlp.sat.thrift.cache.OperatingSystemList; import org.openslx.bwlp.sat.thrift.cache.OrganizationList; @@ -340,9 +342,11 @@ public class ServerHandler implements SatelliteServer.Iface { User.canDeleteImageVersionOrFail(user, imageVersionId); try { DbImage.markForDeletion(imageVersionId); + DbImage.setDeletion(DeleteState.DO_DELETE, imageVersionId); } catch (SQLException e) { throw new TInvocationException(); } + DeleteOldImages.hardDeleteImagesAsync(); } @Override @@ -359,12 +363,23 @@ public class ServerHandler implements SatelliteServer.Iface { String[] ids = new String[imageDetails.versions.size()]; int index = 0; for (ImageVersionDetails version : imageDetails.versions) { - ids[index++] = version.versionId; + if (version.versionId != null) { + ids[index++] = version.versionId; + } + } + if (index != 0) { + try { + DbImage.markForDeletion(ids); + DbImage.setDeletion(DeleteState.DO_DELETE, ids); + } catch (Exception e) { + LOGGER.warn("Could not delete version when trying to delete base image", e); + } + DeleteOldImages.hardDeleteImagesAsync(); } try { - DbImage.markForDeletion(ids); - } catch (Exception e) { - LOGGER.warn("Could not delete version when trying to delete base image", e); + DbImage.deleteBasePermanently(imageBaseId); + } catch (SQLException e) { + throw new TInvocationException(); } } diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/util/Formatter.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/util/Formatter.java index adfeaac3..e55853e8 100644 --- a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/util/Formatter.java +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/util/Formatter.java @@ -1,6 +1,7 @@ package org.openslx.bwlp.sat.util; import java.io.File; +import java.text.Normalizer; import java.util.UUID; import org.joda.time.format.DateTimeFormat; @@ -29,11 +30,12 @@ public class Formatter { * * @param user The user associated with the VM, e.g. the owner * @param imageName Name of the VM + * @param ext * @return File name for the VM derived from the function's input */ - public static String vmName(UserInfo user, String imageName) { + public static String vmName(UserInfo user, String imageName, String ext) { return cleanFileName(vmNameDateFormat.print(System.currentTimeMillis()) + "_" + user.lastName + "_" - + imageName); + + imageName + "." + ext); } /** @@ -47,6 +49,7 @@ public class Formatter { public static String cleanFileName(String name) { if (name == null) return "null"; + name = Normalizer.normalize(name, Normalizer.Form.NFD); name = name.replaceAll("[^a-zA-Z0-9_\\.\\-]+", "_"); if (name.length() > 120) name = name.substring(0, 120); diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebRpc.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebRpc.java index 1cc71e45..7a7d70ea 100644 --- a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebRpc.java +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebRpc.java @@ -15,6 +15,7 @@ import javax.security.auth.login.LoginException; import org.apache.commons.io.output.ByteArrayOutputStream; import org.openslx.bwlp.sat.mail.SmtpMailer; import org.openslx.bwlp.sat.mail.SmtpMailer.EncryptionMode; +import org.openslx.bwlp.sat.maintenance.DeleteOldImages; import org.openslx.util.Util; import fi.iki.elonen.NanoHTTPD; @@ -26,9 +27,20 @@ public class WebRpc { if (uri.equals("mailtest")) { return mailTest(params); } + if (uri.equals("delete-images")) { + return deleteImages(); + } return WebServer.notFound(); } + private static Response deleteImages() { + StringBuilder res = DeleteOldImages.hardDeleteImages(); + if (res == null) + return WebServer.internalServerError(); + return new NanoHTTPD.Response(NanoHTTPD.Response.Status.OK, "text/plain; charset=utf-8", + res.toString()); + } + private static Response mailTest(Map<String, String> params) { SmtpMailer smtpc; String recipient = params.get("recipient"); |