diff options
author | Simon Rettberg | 2015-08-31 18:10:47 +0200 |
---|---|---|
committer | Simon Rettberg | 2015-08-31 18:10:47 +0200 |
commit | d31878bcf8ae7646ec5687e15599866c1bfda94d (patch) | |
tree | ebca77a0d5c19f341507d28b4c64fafc5963fa22 | |
parent | Merge branch 'v1.1' of git.openslx.org:openslx-ng/tutor-module into v1.1 (diff) | |
download | tutor-module-d31878bcf8ae7646ec5687e15599866c1bfda94d.tar.gz tutor-module-d31878bcf8ae7646ec5687e15599866c1bfda94d.tar.xz tutor-module-d31878bcf8ae7646ec5687e15599866c1bfda94d.zip |
[server] SMTP Mailing
16 files changed, 668 insertions, 127 deletions
diff --git a/dozentenmodulserver/setup/sat-01-schema.sql b/dozentenmodulserver/setup/sat-01-schema.sql index dab97282..051ab485 100644 --- a/dozentenmodulserver/setup/sat-01-schema.sql +++ b/dozentenmodulserver/setup/sat-01-schema.sql @@ -196,6 +196,15 @@ CREATE TABLE IF NOT EXISTS `configuration` ( PRIMARY KEY (`parameter`) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; +CREATE TABLE IF NOT EXISTS `mailqueue` ( + `mailid` char(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + `userid` char(36) CHARACTER SET ascii COLLATE ascii_bin NOT NULL, + `message` text NOT NULL, + `failcount` int(11) NOT NULL DEFAULT '0', + `dateline` bigint(20) NOT NULL, + PRIMARY KEY (`mailid`) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + ALTER TABLE `lectureuser` ADD CONSTRAINT `fk_lectureuser_1` FOREIGN KEY (`lectureid`) REFERENCES `sat`.`lecture` (`lectureid`) ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/App.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/App.java index 86d231ef..bdbc8b7f 100644 --- a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/App.java +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/App.java @@ -16,6 +16,7 @@ import org.openslx.bwlp.sat.database.Database; import org.openslx.bwlp.sat.database.mappers.DbImage; import org.openslx.bwlp.sat.fileserv.FileServer; import org.openslx.bwlp.sat.maintenance.DeleteOldImages; +import org.openslx.bwlp.sat.maintenance.MailFlusher; import org.openslx.bwlp.sat.maintenance.SendExpireWarning; import org.openslx.bwlp.sat.thrift.BinaryListener; import org.openslx.bwlp.sat.thrift.cache.OperatingSystemList; @@ -98,6 +99,7 @@ public class App { // Set up maintenance tasks DeleteOldImages.init(); SendExpireWarning.init(); + MailFlusher.init(); // Start Thrift Server Thread t; @@ -132,6 +134,7 @@ public class App { NetRule nn = Json.deserializeThrift(data, NetRule.class); LOGGER.info(nn); } + // Wait for servers for (Thread wait : servers) { boolean success = false; diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbConfiguration.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbConfiguration.java index 27dc2b6e..8fb4497e 100644 --- a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbConfiguration.java +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbConfiguration.java @@ -6,6 +6,7 @@ import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; +import java.nio.charset.StandardCharsets; import java.security.KeyStore; import java.security.KeyStoreException; import java.security.NoSuchAlgorithmException; @@ -18,6 +19,8 @@ import org.apache.log4j.Logger; import org.openslx.bwlp.sat.database.Database; import org.openslx.bwlp.sat.database.MysqlConnection; import org.openslx.bwlp.sat.database.MysqlStatement; +import org.openslx.bwlp.sat.mail.MailQueue.MailConfig; +import org.openslx.bwlp.sat.util.Json; public class DbConfiguration { @@ -25,6 +28,8 @@ public class DbConfiguration { private static final String KEY_CERTIFICATE = "certstore"; + private static final String KEY_MAILCONFIG = "mailconfig"; + public static KeyStore loadKeyStore(String password) throws KeyStoreException, SQLException, NoSuchAlgorithmException, CertificateException, IOException { KeyStore keystore = KeyStore.getInstance("JKS"); @@ -80,4 +85,17 @@ public class DbConfiguration { } } + /** + * Returns mailing configuration (SMTP) from data base. + * + * @return mailing configuration (SMTP) from data base. + * @throws SQLException + */ + public static MailConfig getMailConfig() throws SQLException { + byte[] conf = retrieve(KEY_MAILCONFIG); + if (conf == null) + return null; + return Json.deserialize(new String(conf, StandardCharsets.UTF_8), MailConfig.class); + } + } diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbImage.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbImage.java index c03d8322..95d6380c 100644 --- a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbImage.java +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbImage.java @@ -14,7 +14,7 @@ import org.openslx.bwlp.sat.database.MysqlConnection; import org.openslx.bwlp.sat.database.MysqlStatement; import org.openslx.bwlp.sat.database.Paginator; import org.openslx.bwlp.sat.database.models.LocalImageVersion; -import org.openslx.bwlp.sat.maintenance.Mailer; +import org.openslx.bwlp.sat.mail.MailGenerator; import org.openslx.bwlp.sat.permissions.User; import org.openslx.bwlp.sat.util.FileSystem; import org.openslx.bwlp.sat.util.Util; @@ -701,7 +701,7 @@ public class DbImage { } // Now update the latestversionid of the baseimage if applicable if (setLatestVersion(connection, imageBaseId, latestVersion)) { - Mailer.versionDeleted(imageBaseId, changingVersion, latestVersion); + MailGenerator.versionDeleted(imageBaseId, changingVersion, latestVersion); } } diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbLecture.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbLecture.java index c9f1c6df..8492dd58 100644 --- a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbLecture.java +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbLecture.java @@ -13,7 +13,7 @@ import org.openslx.bwlp.sat.database.Database; import org.openslx.bwlp.sat.database.MysqlConnection; import org.openslx.bwlp.sat.database.MysqlStatement; import org.openslx.bwlp.sat.database.models.LocalImageVersion; -import org.openslx.bwlp.sat.maintenance.Mailer; +import org.openslx.bwlp.sat.mail.MailGenerator; import org.openslx.bwlp.sat.permissions.User; import org.openslx.bwlp.sat.util.Json; import org.openslx.bwlp.sat.util.Util; @@ -344,7 +344,7 @@ public class DbLecture { stmt.setString("imagebaseid", imageBaseId); stmt.executeUpdate(); // Send informative mail to lecture admins - Mailer.lectureAutoUpdate(lectures, newVersion); + MailGenerator.lectureAutoUpdate(lectures, newVersion); } /** @@ -365,14 +365,14 @@ public class DbLecture { if (newVersion == null) { stmt = connection.prepareStatement("UPDATE lecture SET isenabled = 0 WHERE imageversionid = :oldversionid"); stmt.setString("oldversionid", oldVersion.imageVersionId); - Mailer.lectureDeactivated(lectures); + MailGenerator.lectureDeactivated(lectures); } else { // Update and send info mail stmt = connection.prepareStatement("UPDATE lecture SET imageversionid = :newversionid" + " WHERE imageversionid = :oldversionid"); stmt.setString("oldversionid", oldVersion.imageVersionId); stmt.setString("newversionid", newVersion.imageVersionId); - Mailer.lectureForcedUpdate(lectures, newVersion); + MailGenerator.lectureForcedUpdate(lectures, newVersion); } stmt.executeUpdate(); } diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbMailQueue.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbMailQueue.java new file mode 100644 index 00000000..67d3591a --- /dev/null +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbMailQueue.java @@ -0,0 +1,95 @@ +package org.openslx.bwlp.sat.database.mappers; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.log4j.Logger; +import org.openslx.bwlp.sat.database.Database; +import org.openslx.bwlp.sat.database.MysqlConnection; +import org.openslx.bwlp.sat.database.MysqlStatement; +import org.openslx.bwlp.sat.mail.Mail; + +public class DbMailQueue { + + private static final Logger LOGGER = Logger.getLogger(DbMailQueue.class); + + public static void queue(Mail mail) throws SQLException { + try (MysqlConnection connection = Database.getConnection()) { + MysqlStatement stmt = connection.prepareStatement("INSERT INTO mailqueue" + + " (mailid, userid, message, failcount, dateline) VALUES" + + " (:mailid, :userid, :message, 0, UNIX_TIMESTAMP())"); + stmt.setString("mailid", mail.id); + stmt.setString("userid", mail.userId); + stmt.setString("message", mail.message); + stmt.executeUpdate(); + connection.commit(); + } catch (SQLException e) { + LOGGER.error("Query failed in DbMailQueue.queue()", e); + throw e; + } + } + + public static List<Mail> getQueued(int batchSize) throws SQLException { + if (batchSize <= 0) + throw new IllegalArgumentException("batchSize must be > 0"); + try (MysqlConnection connection = Database.getConnection()) { + // Delete old mails that got stuck in the queue, optimize table + MysqlStatement delStmt = connection.prepareStatement("DELETE FROM mailqueue" + + " WHERE UNIX_TIMESTAMP() - dateline > 86400 * 2"); + int cnt = delStmt.executeUpdate(); + if (cnt != 0 || Math.random() < .01) { + MysqlStatement optStmt = connection.prepareStatement("OPTIMIZE TABLE mailqueue"); + optStmt.executeUpdate(); + } + MysqlStatement stmt = connection.prepareStatement("SELECT" + + " mailid, userid, message FROM mailqueue" + + " WHERE failcount < 2 ORDER BY dateline DESC LIMIT " + batchSize); + ResultSet rs = stmt.executeQuery(); + List<Mail> list = new ArrayList<>(); + while (rs.next()) { + list.add(new Mail(rs.getString("mailid"), rs.getString("userid"), rs.getString("message"))); + } + connection.commit(); + return list; + } catch (SQLException e) { + LOGGER.error("Query failed in DbMailQueue.getQueued()", e); + throw e; + } + } + + public static void markFailed(List<Mail> mails) throws SQLException { + if (mails.isEmpty()) + return; + try (MysqlConnection connection = Database.getConnection()) { + MysqlStatement stmt = connection.prepareStatement("UPDATE mailqueue" + + " SET failcount = failcount + 1 WHERE mailid = :mailid"); + for (Mail mail : mails) { + stmt.setString("mailid", mail.id); + stmt.executeUpdate(); + } + connection.commit(); + } catch (SQLException e) { + LOGGER.error("Query failed in DbMailQueue.markFailed()", e); + throw e; + } + } + + public static void markSent(List<Mail> mails) throws SQLException { + if (mails.isEmpty()) + return; + try (MysqlConnection connection = Database.getConnection()) { + MysqlStatement stmt = connection.prepareStatement("DELETE FROM mailqueue WHERE mailid = :mailid"); + for (Mail mail : mails) { + stmt.setString("mailid", mail.id); + stmt.executeUpdate(); + } + connection.commit(); + } catch (SQLException e) { + LOGGER.error("Query failed in DbMailQueue.markFailed()", e); + throw e; + } + } + +} diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbUser.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbUser.java index c0e96296..303b8e15 100644 --- a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbUser.java +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbUser.java @@ -20,34 +20,20 @@ import org.openslx.util.TimeoutHashMap; public class DbUser { - private static final Logger LOGGER = Logger.getLogger(DbUser.class); - - private static Map<String, UserInfo> userCache; + public static class User { + public final UserInfo ui; + public final LocalUser local; - private static void initCache() throws SQLException { - if (userCache != null) - return; - synchronized (DbUser.class) { - if (userCache == null) { - try (MysqlConnection connection = Database.getConnection()) { - userCache = new TimeoutHashMap<>(TimeUnit.DAYS.toMillis(2)); - MysqlStatement stmt = connection.prepareStatement("SELECT userid, firstname, lastname, email, organizationid" - + " FROM user ORDER BY lastlogin DESC LIMIT 30"); - ResultSet rs = stmt.executeQuery(); - while (rs.next()) { - UserInfo user = new UserInfo(rs.getString("userid"), rs.getString("firstname"), - rs.getString("lastname"), rs.getString("email"), - rs.getString("organizationid")); - userCache.put(user.userId, user); - } - } catch (SQLException e) { - LOGGER.error("Query failed in DbUser.initCache()", e); - throw e; - } - } + public User(UserInfo ui, LocalUser local) { + this.ui = ui; + this.local = local; } } + private static final Logger LOGGER = Logger.getLogger(DbUser.class); + + private static Map<String, User> userCache = new TimeoutHashMap<>(TimeUnit.MINUTES.toMillis(15)); + /** * Get all users, starting at page <code>page</code>. * This function will return a maximum of {@link #PER_PAGE} results, so @@ -61,7 +47,8 @@ public class DbUser { if (page < 0) return new ArrayList<>(1); try (MysqlConnection connection = Database.getConnection()) { - MysqlStatement stmt = connection.prepareStatement("SELECT userid, firstname, lastname, email, organizationid" + MysqlStatement stmt = connection.prepareStatement("SELECT userid, firstname, lastname, email, organizationid," + + " lastlogin, canlogin, issuperuser, emailnotifications" + " FROM user ORDER BY userid ASC " + Paginator.limitStatement(page)); ResultSet rs = stmt.executeQuery(); List<UserInfo> list = new ArrayList<>(); @@ -150,30 +137,29 @@ public class DbUser { LOGGER.error("Query failed in DbUser.writeUserOnLogin()", e); throw e; } - synchronized (DbUser.class) { - initCache(); - userCache.put(ui.userId, ui); - } } - public static UserInfo getCached(String userId) throws SQLException, TNotFoundException { + public static User getCached(String userId) throws SQLException, TNotFoundException { synchronized (DbUser.class) { - initCache(); - UserInfo user = userCache.get(userId); + User user = userCache.get(userId); if (user != null) return user; } try (MysqlConnection connection = Database.getConnection()) { - MysqlStatement stmt = connection.prepareStatement("SELECT userid, firstname, lastname, email, organizationid" + MysqlStatement stmt = connection.prepareStatement("SELECT userid, firstname, lastname, email, organizationid," + + " lastlogin, canlogin, issuperuser, emailnotifications" + " FROM user WHERE userid = :userid"); stmt.setString("userid", userId); ResultSet rs = stmt.executeQuery(); if (!rs.next()) throw new TNotFoundException(); - UserInfo user = new UserInfo(rs.getString("userid"), rs.getString("firstname"), + UserInfo userInfo = new UserInfo(rs.getString("userid"), rs.getString("firstname"), rs.getString("lastname"), rs.getString("email"), rs.getString("organizationid")); + LocalUser local = new LocalUser(rs.getLong("lastlogin"), rs.getBoolean("canlogin"), + rs.getBoolean("issuperuser"), rs.getBoolean("emailnotifications")); + User user = new User(userInfo, local); synchronized (DbUser.class) { - userCache.put(user.userId, user); + userCache.put(userInfo.userId, user); } return user; } catch (SQLException e) { @@ -181,5 +167,4 @@ public class DbUser { throw e; } } - } diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/mail/Mail.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/mail/Mail.java new file mode 100644 index 00000000..a795a52e --- /dev/null +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/mail/Mail.java @@ -0,0 +1,15 @@ +package org.openslx.bwlp.sat.mail; + +public class Mail { + + public final String id; + public final String userId; + public final String message; + + public Mail(String id, String userId, String message) { + this.id = id; + this.userId = userId; + this.message = message; + } + +} diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/maintenance/Mailer.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/mail/MailGenerator.java index 23482514..3a976b3d 100644 --- a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/maintenance/Mailer.java +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/mail/MailGenerator.java @@ -1,4 +1,4 @@ -package org.openslx.bwlp.sat.maintenance; +package org.openslx.bwlp.sat.mail; import java.sql.SQLException; import java.util.ArrayList; @@ -8,12 +8,16 @@ import java.util.Map; import java.util.Map.Entry; import org.apache.log4j.Logger; +import org.openslx.bwlp.sat.database.mappers.DbConfiguration; import org.openslx.bwlp.sat.database.mappers.DbImage; import org.openslx.bwlp.sat.database.mappers.DbImagePermissions; import org.openslx.bwlp.sat.database.mappers.DbLecturePermissions; import org.openslx.bwlp.sat.database.mappers.DbUser; +import org.openslx.bwlp.sat.database.mappers.DbUser.User; import org.openslx.bwlp.sat.database.models.LocalImageVersion; +import org.openslx.bwlp.sat.mail.MailQueue.MailConfig; import org.openslx.bwlp.sat.util.Formatter; +import org.openslx.bwlp.sat.util.Util; import org.openslx.bwlp.thrift.iface.ImageDetailsRead; import org.openslx.bwlp.thrift.iface.ImagePermissions; import org.openslx.bwlp.thrift.iface.ImageVersionDetails; @@ -22,11 +26,20 @@ import org.openslx.bwlp.thrift.iface.LectureSummary; import org.openslx.bwlp.thrift.iface.TNotFoundException; import org.openslx.bwlp.thrift.iface.UserInfo; -public class Mailer { +public class MailGenerator { - private static final Logger LOGGER = Logger.getLogger(Mailer.class); + private static final Logger LOGGER = Logger.getLogger(MailGenerator.class); + /** + * Called when an image has been updated, and linked lectures will be moved + * to the new image version. + * + * @param lectures List of affected lectures + * @param newVersion version id of new image + */ public static void lectureAutoUpdate(List<LectureSummary> lectures, LocalImageVersion newVersion) { + if (!hasMailConfig()) + return; for (LectureSummary lecture : lectures) { LOGGER.debug("Auto-Update mail for " + lecture.lectureId + " " + lecture.lectureName); List<UserInfo> relevantUsers = getUserToMail(lecture); @@ -37,6 +50,8 @@ public class Mailer { } public static void lectureForcedUpdate(List<LectureSummary> lectures, LocalImageVersion newVersion) { + if (!hasMailConfig()) + return; for (LectureSummary lecture : lectures) { LOGGER.debug("Forced-Update mail for " + lecture.lectureId + " " + lecture.lectureName); List<UserInfo> relevantUsers = getUserToMail(lecture); @@ -47,6 +62,8 @@ public class Mailer { } public static void lectureDeactivated(List<LectureSummary> lectures) { + if (!hasMailConfig()) + return; for (LectureSummary lecture : lectures) { LOGGER.debug("Deactivated mail for " + lecture.lectureId + " " + lecture.lectureName); List<UserInfo> relevantUsers = getUserToMail(lecture); @@ -56,62 +73,10 @@ public class Mailer { } } - private static List<UserInfo> getUserToMail(LectureSummary lecture) { - Map<String, LecturePermissions> users; - try { - users = DbLecturePermissions.getForLecture(lecture.lectureId, false); - } catch (SQLException e) { - users = new HashMap<>(); - } - users.put(lecture.ownerId, new LecturePermissions(true, true)); - List<UserInfo> list = new ArrayList<>(users.size()); - for (Entry<String, LecturePermissions> entry : users.entrySet()) { - LecturePermissions perms = entry.getValue(); - if (!perms.admin && !perms.edit) - continue; - UserInfo user; - try { - user = DbUser.getCached(entry.getKey()); - } catch (TNotFoundException e) { - LOGGER.warn("UserID " + entry.getKey() + " unknown"); - continue; - } catch (SQLException e) { - continue; // Logging happened in DbUser - } - list.add(user); - } - return list; - } - - private static List<UserInfo> getUserToMail(ImageDetailsRead image) { - Map<String, ImagePermissions> users; - try { - users = DbImagePermissions.getForImageBase(image.imageBaseId, false); - } catch (SQLException e) { - users = new HashMap<>(); - } - users.put(image.ownerId, new ImagePermissions(true, true, true, true)); - List<UserInfo> list = new ArrayList<>(users.size()); - for (Entry<String, ImagePermissions> entry : users.entrySet()) { - ImagePermissions perms = entry.getValue(); - if (!perms.admin && !perms.edit) - continue; - UserInfo user; - try { - user = DbUser.getCached(entry.getKey()); - } catch (TNotFoundException e) { - LOGGER.warn("UserID " + entry.getKey() + " unknown"); - continue; - } catch (SQLException e) { - continue; // Logging happened in DbUser - } - list.add(user); - } - return list; - } - public static void versionDeleted(String imageBaseId, LocalImageVersion oldLocal, LocalImageVersion newLocal) { + if (!hasMailConfig()) + return; ImageDetailsRead image; try { image = DbImage.getImageDetails(null, imageBaseId); @@ -137,8 +102,9 @@ public class Mailer { } else { String uploaderName; try { - UserInfo uploader = DbUser.getCached(newVersion.uploaderId); - uploaderName = uploader.firstName + " " + uploader.lastName + " <" + uploader.eMail + ">"; + User uploader = DbUser.getCached(newVersion.uploaderId); + uploaderName = uploader.ui.firstName + " " + uploader.ui.lastName + " <" + uploader.ui.eMail + + ">"; } catch (TNotFoundException | SQLException e) { uploaderName = "an unknown person"; ; @@ -159,6 +125,8 @@ public class Mailer { } public static void sendDeletionReminder(LocalImageVersion version, int days) { + if (!hasMailConfig()) + return; ImageDetailsRead image; try { image = DbImage.getImageDetails(null, version.imageBaseId); @@ -174,6 +142,8 @@ public class Mailer { } public static void sendDeletionRemainder(LectureSummary lecture, int days) { + if (!hasMailConfig()) + return; List<UserInfo> relevantUsers = getUserToMail(lecture); for (UserInfo user : relevantUsers) { LOGGER.debug("[lecture:" + lecture.lectureName + "] Sending warning mail to " @@ -181,4 +151,77 @@ public class Mailer { } } + public static boolean isValidMailConfig(MailConfig conf) { + return conf != null && conf.port != 0 && !Util.isEmptyString(conf.host) + && !Util.isEmptyString(conf.senderAddress); + } + + private static boolean hasMailConfig() { + MailConfig conf; + try { + conf = DbConfiguration.getMailConfig(); + } catch (SQLException e) { + return false; + } + return isValidMailConfig(conf); + } + + private static List<UserInfo> getUserToMail(LectureSummary lecture) { + Map<String, LecturePermissions> users; + try { + users = DbLecturePermissions.getForLecture(lecture.lectureId, false); + } catch (SQLException e) { + users = new HashMap<>(); + } + users.put(lecture.ownerId, new LecturePermissions(true, true)); + List<UserInfo> list = new ArrayList<>(users.size()); + for (Entry<String, LecturePermissions> entry : users.entrySet()) { + LecturePermissions perms = entry.getValue(); + if (!perms.admin && !perms.edit) + continue; + User user; + try { + user = DbUser.getCached(entry.getKey()); + } catch (TNotFoundException e) { + LOGGER.warn("UserID " + entry.getKey() + " unknown"); + continue; + } catch (SQLException e) { + continue; // Logging happened in DbUser + } + if (user.local.emailNotifications) { + list.add(user.ui); + } + } + return list; + } + + private static List<UserInfo> getUserToMail(ImageDetailsRead image) { + Map<String, ImagePermissions> users; + try { + users = DbImagePermissions.getForImageBase(image.imageBaseId, false); + } catch (SQLException e) { + users = new HashMap<>(); + } + users.put(image.ownerId, new ImagePermissions(true, true, true, true)); + List<UserInfo> list = new ArrayList<>(users.size()); + for (Entry<String, ImagePermissions> entry : users.entrySet()) { + ImagePermissions perms = entry.getValue(); + if (!perms.admin && !perms.edit) + continue; + User user; + try { + user = DbUser.getCached(entry.getKey()); + } catch (TNotFoundException e) { + LOGGER.warn("UserID " + entry.getKey() + " unknown"); + continue; + } catch (SQLException e) { + continue; // Logging happened in DbUser + } + if (user.local.emailNotifications) { + list.add(user.ui); + } + } + return list; + } + } diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/mail/MailQueue.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/mail/MailQueue.java new file mode 100644 index 00000000..ae1932e1 --- /dev/null +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/mail/MailQueue.java @@ -0,0 +1,206 @@ +package org.openslx.bwlp.sat.mail; + +import java.io.IOException; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; +import java.security.spec.InvalidKeySpecException; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import javax.security.auth.login.LoginException; + +import org.apache.log4j.Logger; +import org.openslx.bwlp.sat.database.mappers.DbConfiguration; +import org.openslx.bwlp.sat.database.mappers.DbMailQueue; +import org.openslx.bwlp.sat.database.mappers.DbUser; +import org.openslx.bwlp.sat.database.mappers.DbUser.User; +import org.openslx.bwlp.sat.mail.SmtpMailer.EncryptionMode; +import org.openslx.bwlp.sat.maintenance.MailFlusher; +import org.openslx.bwlp.sat.util.Util; +import org.openslx.bwlp.thrift.iface.TNotFoundException; +import org.openslx.bwlp.thrift.iface.UserInfo; +import org.openslx.util.QuickTimer; +import org.openslx.util.QuickTimer.Task; + +public class MailQueue { + + public static class MailConfig { + public String host; + public int port; + public EncryptionMode ssl; + public String senderAddress; + public String replyTo; + public String password; + public String username; + public String serverName; + } + + private static final Logger LOGGER = Logger.getLogger(MailQueue.class); + + private static final int BATCH_SIZE = 25; + + private static boolean busy = false; + + /** + * Convenience wrapper for {@link DbMailQueue#queue(Mail)}, swallowing any + * {@link SQLException}, so the mailing will keep going (or try to at least) + * + * @param mail Mail to queue for sending + */ + public static void queue(Mail mail) { + try { + DbMailQueue.queue(mail); + } catch (SQLException e) { + } + } + + /** + * Send queued messages. This operation is rate-limited. In case there might + * be more messages to send, this function will return after sending some of + * them, and schedule a maintenance job that will trigger this method again. + * + * @throws InterruptedException + */ + public static void flush() throws InterruptedException { + synchronized (MailQueue.class) { + if (busy) // Will run again when the scheduler decides to + return; + busy = true; + } + try { + List<Mail> queuedMails; + try { + queuedMails = DbMailQueue.getQueued(BATCH_SIZE); + } catch (SQLException e) { + LOGGER.error("Cannot retrieve queued mails from DB"); + return; + } + // Anything to do? + if (queuedMails.isEmpty()) + return; + // Fetch SMTP config + MailConfig conf; + try { + conf = DbConfiguration.getMailConfig(); + } catch (SQLException e) { + conf = null; + return; + } + if (!MailGenerator.isValidMailConfig(conf)) { + LOGGER.error("Cannot send mail with no mail config"); + return; + } + // Setup mailer + SmtpMailer smtpc; + try { + smtpc = new SmtpMailer(conf.host, conf.port, conf.ssl, conf.senderAddress, conf.serverName, + conf.replyTo, conf.username, conf.password); + } catch (InvalidKeyException | LoginException | NoSuchAlgorithmException + | InvalidKeySpecException | IOException e) { + LOGGER.error("Could not initialize connection to SMTP server. Mails will not be sent", e); + return; + } + // Iterate over mail: Group by receiving user + Map<String, List<Mail>> batch = new HashMap<>(); + for (Mail mail : queuedMails) { + List<Mail> list = batch.get(mail.userId); + if (list == null) { + list = new ArrayList<>(); + batch.put(mail.userId, list); + } + list.add(mail); + } + // Send all the mails + int delaySeconds = 2; + boolean sendOk = true; + for (List<Mail> userBatch : batch.values()) { + if (userBatch.isEmpty()) { + continue; // Now how the hell did that happen? + } + User cachedUser; + try { + cachedUser = DbUser.getCached(userBatch.get(0).userId); + } catch (TNotFoundException | SQLException e) { + LOGGER.warn("Cannot get user for id " + userBatch.get(0).userId + + ": Sending mails failed."); + try { + DbMailQueue.markFailed(userBatch); + } catch (SQLException e1) { + } + continue; + } + // Double-check if user wants mail (unlikely, but user might just have changed the setting) + if (!cachedUser.local.emailNotifications) { + try { + DbMailQueue.markSent(userBatch); + } catch (SQLException e) { + } + continue; + } + StringBuilder sb = new StringBuilder(); + for (Mail mail : userBatch) { + if (sb.length() != 0) { + sb.append('\n'); + } + sb.append("* "); + sb.append(mail.message); + } + sendOk = sendMail(conf, smtpc, cachedUser.ui, sb.toString()); + LOGGER.debug("Sending mail to " + cachedUser.ui.eMail + ": " + + (sendOk ? "success" : "failure")); + try { + if (sendOk) { + DbMailQueue.markSent(userBatch); + } else { + DbMailQueue.markFailed(userBatch); + } + } catch (SQLException e) { + } + Thread.sleep(delaySeconds * 1000); + delaySeconds += 1; + } + smtpc.close(); + // We got a full batch from DB, call flush() again in a minute + if (queuedMails.size() == BATCH_SIZE && sendOk) { + callAgainInOneMinute(); + } + } finally { + synchronized (MailQueue.class) { + busy = false; + } + } + } + + private static void callAgainInOneMinute() { + QuickTimer.scheduleOnce(new Task() { + @Override + public void fire() { + MailFlusher.start(); + } + }, TimeUnit.MINUTES.toMillis(1)); + } + + private static boolean sendMail(MailConfig conf, SmtpMailer smtpc, UserInfo user, String message) { + // TODO: Template + String fullMessage = "Guten Tag " + user.firstName + " " + user.lastName + ",\n\n" + + "Bitte beachten Sie folgende Hinweise zu Virtuellen Maschinen und Veranstaltungen,\n" + + "für die Sie als zuständige Person hinterlegt sind:\n\n"; + fullMessage += message; + fullMessage += "\n\n" + + "Dies ist eine automatisch generierte Mail. Wenn Sie keine Hinweise dieser Art\n" + + "wünschen, melden Sie sich bitte mittels des Desktop-Clients an diesem\n" + + "bwLehrpool Satellitenserver an, und deaktivieren Sie in den Einstellungen\n" + + "e-Mail-Benachrichtigungen."; + if (!Util.isEmptyString(conf.replyTo)) { + fullMessage += "\n\nBei weiteren Fragen wenden Sie sich bitte an den Support unter\n" + + conf.replyTo; + } + fullMessage += "\n\n-----------------------------\n" + "Generiert von " + conf.serverName; + return smtpc.send(user.eMail, "[bwLehrstuhl] Hinweise zu Ihren VMs/Veranstaltungen", fullMessage); + } + +} diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/mail/QuotingSmtpHeader.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/mail/QuotingSmtpHeader.java new file mode 100644 index 00000000..35be803a --- /dev/null +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/mail/QuotingSmtpHeader.java @@ -0,0 +1,56 @@ +package org.openslx.bwlp.sat.mail; + +import java.nio.charset.CharsetEncoder; +import java.nio.charset.StandardCharsets; + +import org.apache.commons.codec.binary.Base64; +import org.apache.commons.net.smtp.SimpleSMTPHeader; + +public class QuotingSmtpHeader extends SimpleSMTPHeader { + + private static CharsetEncoder asciiEncoder = StandardCharsets.US_ASCII.newEncoder(); + + public QuotingSmtpHeader(String fromAddress, String fromDisplayName, String to, String subject) { + super(buildNamedAddress(fromAddress, fromDisplayName), to, wrapEncoding(subject)); + } + + public QuotingSmtpHeader(String from, String to, String subject) { + super(from, to, wrapEncoding(subject)); + } + + @Override + public void addHeaderField(String headerField, String value) { + super.addHeaderField(headerField, wrapEncoding(value)); + } + + @Override + public void addCC(String address) { + super.addCC(wrapEncoding(address)); + } + + private static String wrapEncoding(String input) { + return wrapEncoding(input, false); + } + + private static String wrapEncoding(String input, boolean addQuotesIfSpaces) { + boolean isAscii; + synchronized (asciiEncoder) { // Has class-wide state vars + isAscii = asciiEncoder.canEncode(input); + } + if (isAscii) { + if (addQuotesIfSpaces && (input.contains(" ") || input.contains("\t"))) + return "\"" + input + "\""; + return input; + } + return "=?utf-8?B?" + + new String(Base64.encodeBase64(input.getBytes(StandardCharsets.UTF_8), false), + StandardCharsets.UTF_8) + "?="; + } + + private static String buildNamedAddress(String address, String name) { + if (name == null) + return address; + return wrapEncoding(name, true) + " <" + address + ">"; + } + +} diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/maintenance/SmtpMailer.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/mail/SmtpMailer.java index 52c6d9b6..4f1b415e 100644 --- a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/maintenance/SmtpMailer.java +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/mail/SmtpMailer.java @@ -1,4 +1,4 @@ -package org.openslx.bwlp.sat.maintenance; +package org.openslx.bwlp.sat.mail; import java.io.IOException; import java.io.Writer; @@ -11,14 +11,20 @@ import java.security.spec.InvalidKeySpecException; import javax.security.auth.login.LoginException; +import org.apache.commons.net.PrintCommandListener; +import org.apache.commons.net.ProtocolCommandEvent; +import org.apache.commons.net.ProtocolCommandListener; import org.apache.commons.net.smtp.AuthenticatingSMTPClient; import org.apache.commons.net.smtp.AuthenticatingSMTPClient.AUTH_METHOD; import org.apache.commons.net.smtp.SMTPReply; import org.apache.commons.net.smtp.SimpleSMTPHeader; +import org.apache.log4j.Logger; +import org.openslx.bwlp.sat.util.Util; public class SmtpMailer { // TODO Logging + private static final Logger LOGGER = Logger.getLogger(SmtpMailer.class); public enum EncryptionMode { NONE, @@ -26,16 +32,19 @@ public class SmtpMailer { EXPLICIT } - private final String from; + private final String fromAddress; + private final String fromName; private final String replyTo; private final AuthenticatingSMTPClient client; - public SmtpMailer(String host, int port, EncryptionMode ssl, String from, String replyTo, - String username, String password) throws UnknownHostException, SocketException, IOException, - LoginException, InvalidKeyException, NoSuchAlgorithmException, InvalidKeySpecException { + public SmtpMailer(String host, int port, EncryptionMode ssl, String fromAddress, String fromName, + String replyTo, String username, String password) throws UnknownHostException, SocketException, + IOException, LoginException, InvalidKeyException, NoSuchAlgorithmException, + InvalidKeySpecException { InetAddress[] ips = InetAddress.getAllByName(host); if (ips == null || ips.length == 0) throw new UnknownHostException(host); + LOGGER.debug("Mailing via " + host + ", " + ssl); if (ssl == EncryptionMode.EXPLICIT || ssl == EncryptionMode.NONE) { client = new AuthenticatingSMTPClient("TLSv1.2", false, "UTF-8"); } else { @@ -43,6 +52,20 @@ public class SmtpMailer { } boolean cleanup = true; try { + new ProtocolCommandListener() { + + @Override + public void protocolReplyReceived(ProtocolCommandEvent event) { + event.getMessage(); + } + + @Override + public void protocolCommandSent(ProtocolCommandEvent event) { + // TODO Auto-generated method stub + + } + }; + client.addProtocolCommandListener(new PrintCommandListener(System.out)); client.setConnectTimeout(5000); IOException conEx = null; for (InetAddress ip : ips) { @@ -66,17 +89,20 @@ public class SmtpMailer { if (ssl == EncryptionMode.EXPLICIT && !client.execTLS()) { throw new LoginException("STARTTLS (explicit TLS) failed"); } - boolean authed = false; - try { - authed = client.auth(AUTH_METHOD.CRAM_MD5, username, password); - } catch (InvalidKeyException | NoSuchAlgorithmException | InvalidKeySpecException e) { - e.printStackTrace(); - } - if (!authed && !client.auth(AUTH_METHOD.PLAIN, username, password)) { - throw new LoginException("Server rejected AUTH command. Invalid username or password?"); + if (!Util.isEmptyString(username)) { + boolean authed = false; + try { + authed = client.auth(AUTH_METHOD.CRAM_MD5, username, password); + } catch (InvalidKeyException | NoSuchAlgorithmException | InvalidKeySpecException e) { + e.printStackTrace(); + } + if (!authed && !client.auth(AUTH_METHOD.PLAIN, username, password)) { + throw new LoginException("Server rejected AUTH command. Invalid username or password?"); + } } cleanup = false; - this.from = from; + this.fromAddress = fromAddress; + this.fromName = fromName; this.replyTo = replyTo; } finally { if (cleanup) @@ -105,12 +131,14 @@ public class SmtpMailer { SimpleSMTPHeader header; try { - header = new SimpleSMTPHeader(from, recipient, subject); - if (replyTo != null && !replyTo.isEmpty()) { + header = new QuotingSmtpHeader(fromAddress, fromName, recipient, subject); + if (!Util.isEmptyString(replyTo)) { header.addHeaderField("Reply-To", replyTo); } + header.addHeaderField("Content-Type", "text/plain; charset=utf-8"); + header.addHeaderField("Content-Transfer-Encoding", "8bit"); - if (!client.setSender(from)) { + if (!client.setSender(fromAddress)) { abort(); return false; } @@ -147,4 +175,17 @@ public class SmtpMailer { } } + public void close() { + if (client.isConnected()) { + try { + client.logout(); + } catch (Exception e) { // Don't care + } + try { + client.disconnect(); + } catch (Exception e) { // Don't care + } + } + } + } diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/maintenance/MailFlusher.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/maintenance/MailFlusher.java new file mode 100644 index 00000000..ca4003b4 --- /dev/null +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/maintenance/MailFlusher.java @@ -0,0 +1,58 @@ +package org.openslx.bwlp.sat.maintenance; + +import java.util.concurrent.TimeUnit; + +import org.apache.log4j.Logger; +import org.joda.time.DateTime; +import org.openslx.bwlp.sat.mail.MailQueue; +import org.openslx.util.QuickTimer; +import org.openslx.util.QuickTimer.Task; + +public class MailFlusher implements Runnable { + + private static final Logger LOGGER = Logger.getLogger(MailFlusher.class); + + private static final MailFlusher instance = new MailFlusher(); + + private static long blockedUntil = 0; + + /** + * Initialize the task. This schedules a timer that runs + * every 5 minutes. If the hour of day reaches 3, it will fire + * the task, and block it from running for the next 12 hours. + */ + public synchronized static void init() { + if (blockedUntil != 0) + return; + blockedUntil = 1; + QuickTimer.scheduleAtFixedRate(new Task() { + @Override + public void fire() { + if (blockedUntil > System.currentTimeMillis()) + return; + DateTime now = DateTime.now(); + if (now.getHourOfDay() != 3 || now.getMinuteOfHour() > 15) + return; + start(); + } + }, TimeUnit.MINUTES.toMillis(6), TimeUnit.MINUTES.toMillis(10)); + } + + public synchronized static void start() { + if (blockedUntil > System.currentTimeMillis()) + return; + if (Maintenance.trySubmit(instance)) { + blockedUntil = System.currentTimeMillis() + TimeUnit.MINUTES.toMillis(1); + } + } + + @Override + public void run() { + try { + MailQueue.flush(); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + +} diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/maintenance/SendExpireWarning.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/maintenance/SendExpireWarning.java index 96436471..187b7628 100644 --- a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/maintenance/SendExpireWarning.java +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/maintenance/SendExpireWarning.java @@ -9,6 +9,7 @@ import org.joda.time.DateTime; import org.openslx.bwlp.sat.database.mappers.DbImage; import org.openslx.bwlp.sat.database.mappers.DbLecture; import org.openslx.bwlp.sat.database.models.LocalImageVersion; +import org.openslx.bwlp.sat.mail.MailGenerator; import org.openslx.bwlp.sat.util.FileSystem; import org.openslx.bwlp.sat.util.Util; import org.openslx.bwlp.thrift.iface.LectureSummary; @@ -73,7 +74,7 @@ public class SendExpireWarning implements Runnable { final int days = (int) ((lecture.endTime - now) / 86400); LOGGER.debug(lecture.lectureName + " expires in " + days); if ((lecture.isEnabled && (days == 14 || days == 1)) || (days == 7)) { - Mailer.sendDeletionRemainder(lecture, days); + MailGenerator.sendDeletionRemainder(lecture, days); } } } @@ -99,7 +100,7 @@ public class SendExpireWarning implements Runnable { LOGGER.debug(version.imageVersionId + " expires in " + days); if ((version.isValid && (days == 14 || days == 7 || days == 1)) || (!version.isValid && days == 3)) { - Mailer.sendDeletionReminder(version, days); + MailGenerator.sendDeletionReminder(version, days); } } } diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/util/Json.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/util/Json.java index bb222792..679cb6bf 100644 --- a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/util/Json.java +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/util/Json.java @@ -16,6 +16,7 @@ import com.google.gson.JsonDeserializer; import com.google.gson.JsonElement; import com.google.gson.JsonObject; import com.google.gson.JsonParseException; +import com.google.gson.JsonSyntaxException; public class Json { @@ -43,11 +44,21 @@ public class Json { * @return instanceof T */ public static <T> T deserialize(String data, Class<T> classOfData) { - return gson.fromJson(data, classOfData); + try { + return gson.fromJson(data, classOfData); + } catch (JsonSyntaxException e) { + LOGGER.warn("Cannot deserialize to " + classOfData.getSimpleName(), e); + return null; + } } public static <T> T deserializeThrift(String data, Class<T> thriftClass) { - return gsonThriftBuilder.create().fromJson(data, thriftClass); + try { + return gsonThriftBuilder.create().fromJson(data, thriftClass); + } catch (JsonSyntaxException e) { + LOGGER.warn("Cannot deserialize to " + thriftClass.getSimpleName(), e); + return null; + } } /** diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/util/Util.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/util/Util.java index e2135e64..53bfa403 100644 --- a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/util/Util.java +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/util/Util.java @@ -43,8 +43,8 @@ public class Util { } } - private static Pattern stringChecker = Pattern.compile("[\\p{C}\\p{Zl}\\p{Zp}]"); - private static Pattern nonSpaceMatcher = Pattern.compile("[^\\p{C}\\p{Z}]"); + private static Pattern nonprintableExp = Pattern.compile("[\\p{C}\\p{Zl}\\p{Zp}]"); + private static Pattern nonSpaceExp = Pattern.compile("[^\\p{C}\\p{Z}]"); /** * Whether the given string contains only printable characters. @@ -53,13 +53,13 @@ public class Util { * @return */ public static boolean isPrintable(String string) { - return !stringChecker.matcher(string).find(); + return !nonprintableExp.matcher(string).find(); } public static boolean isEmptyString(String string) { - return !nonSpaceMatcher.matcher(string).find(); + return string == null || !nonSpaceExp.matcher(string).find(); } - + public static long unixTime() { return System.currentTimeMillis() / 1000; } |