diff options
Diffstat (limited to 'dozentenmodulserver/src/main/java/org/openslx/bwlp')
23 files changed, 1922 insertions, 0 deletions
diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/App.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/App.java new file mode 100644 index 00000000..8aac1fcb --- /dev/null +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/App.java @@ -0,0 +1,93 @@ +package org.openslx.bwlp.sat; + +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.TimerTask; + +import org.apache.log4j.BasicConfigurator; +import org.apache.log4j.Logger; +import org.apache.thrift.transport.TTransportException; +import org.openslx.bwlp.sat.database.Database; +import org.openslx.bwlp.sat.database.mappers.DbImage; +import org.openslx.bwlp.sat.fileserv.FileServer; +import org.openslx.bwlp.sat.thrift.BinaryListener; +import org.openslx.bwlp.sat.thrift.cache.OperatingSystemList; +import org.openslx.bwlp.sat.thrift.cache.OrganizationList; +import org.openslx.bwlp.sat.util.Configuration; +import org.openslx.bwlp.sat.util.QuickTimer; +import org.openslx.bwlp.thrift.iface.ImageSummaryRead; +import org.openslx.bwlp.thrift.iface.UserInfo; + +public class App { + + private static Logger log = Logger.getLogger(App.class); + + private static List<Thread> servers = new ArrayList<>(); + + public static boolean DEBUG = false; + + public static void main(String[] args) throws TTransportException, NoSuchAlgorithmException { + //get going and show basic information in log file + BasicConfigurator.configure(); + if (args.length != 0 && args[0].equals("debug")) { + DEBUG = true; + } + log.info("****************************************************************"); + log.info("******************* Starting Application ***********************"); + log.info("****************************************************************"); + + // get Configuration + try { + log.info("Loading configuration"); + Configuration.load(); + } catch (Exception e1) { + log.fatal("Could not load configuration", e1); + System.exit(1); + } + + // Load useful things from master server + OrganizationList.get(); + //OperatingSystemList.get(); + + // Start file transfer server + if (!FileServer.instance().start()) { + log.error("Could not start internal file server."); + return; + } + // Start Server + Thread t; + t = new Thread(new BinaryListener(9090, false)); + servers.add(t); + t.start(); + // DEBUG + if (DEBUG) { + Database.printCharsetInformation(); + List<ImageSummaryRead> allVisible = DbImage.getAllVisible(new UserInfo("bla", "blu", null, null, + null), null); + log.info("Got " + allVisible.size()); + QuickTimer.scheduleAtFixedDelay(new TimerTask() { + @Override + public void run() { + Database.printDebug(); + } + }, 100, 5000); + } + // Wait for servers + for (Thread wait : servers) { + boolean success = false; + while (!success) { + try { + wait.join(); + success = true; + } catch (InterruptedException e) { + // Do nothing... + } + } + } + QuickTimer.cancel(); + log.info(new Date() + " - all Servers shut down, exiting...\n"); + } + +} diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/Database.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/Database.java new file mode 100644 index 00000000..cfc6530b --- /dev/null +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/Database.java @@ -0,0 +1,120 @@ +package org.openslx.bwlp.sat.database;
+
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.Collections;
+import java.util.Queue;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentLinkedQueue;
+
+import org.apache.log4j.Logger;
+import org.openslx.bwlp.sat.util.Configuration;
+
+public class Database {
+
+ private static final Logger LOGGER = Logger.getLogger(Database.class);
+ /**
+ * Pool of available connections.
+ */
+ private static final Queue<MysqlConnection> pool = new ConcurrentLinkedQueue<>();
+
+ /**
+ * Set of connections currently handed out.
+ */
+ private static final Set<MysqlConnection> busyConnections = Collections.newSetFromMap(new ConcurrentHashMap<MysqlConnection, Boolean>());
+
+ static {
+ try {
+ Class.forName("com.mysql.jdbc.Driver").newInstance();
+ } catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) {
+ LOGGER.fatal("Cannot get mysql JDBC driver!", e);
+ System.exit(1);
+ }
+ }
+
+ /**
+ * Get a connection to the database. If there is a valid connection in the
+ * pool, it will be returned. Otherwise, a new connection is created. If
+ * there are more than 20 busy connections, <code>null</code> is returned.
+ *
+ * @return connection to database, or <code>null</code>
+ */
+ public static MysqlConnection getConnection() {
+ MysqlConnection con;
+ for (;;) {
+ con = pool.poll();
+ if (con == null)
+ break;
+ if (!con.isValid()) {
+ con.release();
+ continue;
+ }
+ if (!busyConnections.add(con))
+ throw new RuntimeException("Tried to hand out a busy connection!");
+ return con;
+ }
+ // No pooled connection
+ if (busyConnections.size() > 20) {
+ LOGGER.warn("Too many open MySQL connections. Possible connection leak!");
+ return null;
+ }
+ try {
+ // Create fresh connection
+ Connection rawConnection = DriverManager.getConnection(Configuration.getDbUri(),
+ Configuration.getDbUsername(), Configuration.getDbPassword());
+ // By convention in our program we don't want auto commit
+ rawConnection.setAutoCommit(false);
+ // Wrap into our proxy
+ con = new MysqlConnection(rawConnection);
+ // Keep track of busy mysql connection
+ if (!busyConnections.add(con))
+ throw new RuntimeException("Tried to hand out a busy connection!");
+ return con;
+ } catch (SQLException e) {
+ LOGGER.info("Failed to connect to local mysql server", e);
+ }
+ return null;
+ }
+
+ /**
+ * Called by a {@link MysqlConnection} when its <code>close()</code>-method
+ * is called, so the connection will be added to the pool of available
+ * connections again.
+ *
+ * @param connection
+ */
+ static void returnConnection(MysqlConnection connection) {
+ if (!busyConnections.remove(connection))
+ throw new RuntimeException("Tried to return a mysql connection to the pool that was not taken!");
+ pool.add(connection);
+ }
+
+ public static void printCharsetInformation() {
+ LOGGER.info("MySQL charset related variables:");
+ try (MysqlConnection connection = Database.getConnection()) {
+ MysqlStatement stmt = connection.prepareStatement("SHOW VARIABLES LIKE :what");
+ stmt.setString("what", "char%");
+ ResultSet rs = stmt.executeQuery();
+ while (rs.next()) {
+ LOGGER.info(rs.getString("Variable_name") + ": " + rs.getString("Value"));
+ }
+ stmt.setString("what", "collat%");
+ rs = stmt.executeQuery();
+ while (rs.next()) {
+ LOGGER.info(rs.getString("Variable_name") + ": " + rs.getString("Value"));
+ }
+ } catch (SQLException e) {
+ LOGGER.error("Query failed in Database.printCharsetInformation()", e);
+ }
+ LOGGER.info("End of variables");
+ }
+
+ public static void printDebug() {
+ LOGGER.info("Available: " + pool.size());
+ LOGGER.info("Busy: " + busyConnections.size());
+ }
+
+}// end class
diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/MysqlConnection.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/MysqlConnection.java new file mode 100644 index 00000000..24aaf1e8 --- /dev/null +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/MysqlConnection.java @@ -0,0 +1,77 @@ +package org.openslx.bwlp.sat.database; + +import java.sql.Connection; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.log4j.Logger; + +public class MysqlConnection implements AutoCloseable { + + private static final Logger LOGGER = Logger.getLogger(MysqlConnection.class); + + private static final int CONNECTION_TIMEOUT_MS = 5 * 60 * 1000; + + private final long deadline = System.currentTimeMillis() + CONNECTION_TIMEOUT_MS; + + private final Connection rawConnection; + + private boolean hasPendingQueries = false; + + private List<MysqlStatement> openStatements = new ArrayList<>(); + + MysqlConnection(Connection rawConnection) { + this.rawConnection = rawConnection; + } + + public MysqlStatement prepareStatement(String sql) throws SQLException { + if (!sql.startsWith("SELECT")) + hasPendingQueries = true; + MysqlStatement statement = new MysqlStatement(rawConnection, sql); + openStatements.add(statement); + return statement; + } + + public void commit() throws SQLException { + rawConnection.commit(); + hasPendingQueries = false; + } + + public void rollback() throws SQLException { + rawConnection.rollback(); + hasPendingQueries = false; + } + + boolean isValid() { + return System.currentTimeMillis() < deadline; + } + + @Override + public void close() { + if (hasPendingQueries) { + LOGGER.warn("Mysql connection had uncommited queries on .close()"); + try { + rawConnection.rollback(); + } catch (SQLException e) { + LOGGER.warn("Rolling back uncommited queries failed!", e); + } + } + if (!openStatements.isEmpty()) { + for (MysqlStatement statement : openStatements) { + statement.close(); + } + openStatements.clear(); + } + Database.returnConnection(this); + } + + void release() { + try { + rawConnection.close(); + } catch (SQLException e) { + // Nothing meaningful to do + } + } + +} diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/MysqlStatement.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/MysqlStatement.java new file mode 100644 index 00000000..3d5f9065 --- /dev/null +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/MysqlStatement.java @@ -0,0 +1,289 @@ +package org.openslx.bwlp.sat.database; + +import java.io.Closeable; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Class for creating {@link PreparedStatement}s with named parameters. Based on + * <a href= + * "http://www.javaworld.com/article/2077706/core-java/named-parameters-for-preparedstatement.html?page=2" + * >Named Parameters for PreparedStatement</a> + */ +public class MysqlStatement implements Closeable { + + private static final QueryCache cache = new QueryCache(); + + private final PreparsedQuery query; + + private final PreparedStatement statement; + + private final List<ResultSet> openResultSets = new ArrayList<>(); + + MysqlStatement(Connection con, String sql) throws SQLException { + PreparsedQuery query; + synchronized (cache) { + query = cache.get(sql); + } + if (query == null) { + query = parse(sql); + synchronized (cache) { + cache.put(sql, query); + } + } + this.query = query; + this.statement = con.prepareStatement(query.sql); + } + + /** + * Returns the indexes for a parameter. + * + * @param name parameter name + * @return parameter indexes + * @throws IllegalArgumentException if the parameter does not exist + */ + private List<Integer> getIndexes(String name) { + List<Integer> indexes = query.indexMap.get(name); + if (indexes == null) { + throw new IllegalArgumentException("Parameter not found: " + name); + } + return indexes; + } + + /** + * Sets a parameter. + * + * @param name parameter name + * @param value parameter value + * @throws SQLException if an error occurred + * @throws IllegalArgumentException if the parameter does not exist + * @see PreparedStatement#setObject(int, java.lang.Object) + */ + public void setObject(String name, Object value) throws SQLException { + List<Integer> indexes = getIndexes(name); + for (Integer index : indexes) { + statement.setObject(index, value); + } + } + + /** + * Sets a parameter. + * + * @param name parameter name + * @param value parameter value + * @throws SQLException if an error occurred + * @throws IllegalArgumentException if the parameter does not exist + * @see PreparedStatement#setString(int, java.lang.String) + */ + public void setString(String name, String value) throws SQLException { + List<Integer> indexes = getIndexes(name); + for (Integer index : indexes) { + statement.setString(index, value); + } + } + + /** + * Sets a parameter. + * + * @param name parameter name + * @param value parameter value + * @throws SQLException if an error occurred + * @throws IllegalArgumentException if the parameter does not exist + * @see PreparedStatement#setInt(int, int) + */ + public void setInt(String name, int value) throws SQLException { + List<Integer> indexes = getIndexes(name); + for (Integer index : indexes) { + statement.setInt(index, value); + } + } + + /** + * Sets a parameter. + * + * @param name parameter name + * @param value parameter value + * @throws SQLException if an error occurred + * @throws IllegalArgumentException if the parameter does not exist + * @see PreparedStatement#setInt(int, int) + */ + public void setLong(String name, long value) throws SQLException { + List<Integer> indexes = getIndexes(name); + for (Integer index : indexes) { + statement.setLong(index, value); + } + } + + /** + * Executes the statement. + * + * @return true if the first result is a {@link ResultSet} + * @throws SQLException if an error occurred + * @see PreparedStatement#execute() + */ + public boolean execute() throws SQLException { + return statement.execute(); + } + + /** + * Executes the statement, which must be a query. + * + * @return the query results + * @throws SQLException if an error occurred + * @see PreparedStatement#executeQuery() + */ + public ResultSet executeQuery() throws SQLException { + ResultSet rs = statement.executeQuery(); + openResultSets.add(rs); + return rs; + } + + /** + * Executes the statement, which must be an SQL INSERT, UPDATE or DELETE + * statement; or an SQL statement that returns nothing, such as a DDL + * statement. + * + * @return number of rows affected + * @throws SQLException if an error occurred + * @see PreparedStatement#executeUpdate() + */ + public int executeUpdate() throws SQLException { + return statement.executeUpdate(); + } + + /** + * Closes the statement. + * + * @see Statement#close() + */ + @Override + public void close() { + for (ResultSet rs : openResultSets) { + try { + rs.close(); + } catch (SQLException e) { + // + } + } + try { + statement.close(); + } catch (SQLException e) { + // Nothing to do + } + } + + /** + * Adds the current set of parameters as a batch entry. + * + * @throws SQLException if something went wrong + */ + public void addBatch() throws SQLException { + statement.addBatch(); + } + + /** + * Executes all of the batched statements. + * + * See {@link Statement#executeBatch()} for details. + * + * @return update counts for each statement + * @throws SQLException if something went wrong + */ + public int[] executeBatch() throws SQLException { + return statement.executeBatch(); + } + + // static methods + + private static PreparsedQuery parse(String query) { + int length = query.length(); + StringBuffer parsedQuery = new StringBuffer(length); + Map<String, List<Integer>> paramMap = new HashMap<>(); + boolean inSingleQuote = false; + boolean inDoubleQuote = false; + boolean hasBackslash = false; + int index = 1; + + for (int i = 0; i < length; i++) { + char c = query.charAt(i); + if (hasBackslash) { + // Last char was a backslash, so we ignore the current char + hasBackslash = false; + } else if (c == '\\') { + // This is a backslash, next char will be escaped + hasBackslash = true; + } else if (inSingleQuote) { + // End of quoted string + if (c == '\'') { + inSingleQuote = false; + } + } else if (inDoubleQuote) { + // End of quoted string + if (c == '"') { + inDoubleQuote = false; + } + } else { + // Not in string, look for named params + if (c == '\'') { + inSingleQuote = true; + } else if (c == '"') { + inDoubleQuote = true; + } else if (c == ':' && i + 1 < length && Character.isJavaIdentifierStart(query.charAt(i + 1))) { + int j = i + 2; + while (j < length && Character.isJavaIdentifierPart(query.charAt(j))) { + j++; + } + String name = query.substring(i + 1, j); + c = '?'; // replace the parameter with a question mark + i += name.length(); // skip past the end of the parameter + + List<Integer> indexList = paramMap.get(name); + if (indexList == null) { + indexList = new ArrayList<>(); + paramMap.put(name, indexList); + } + indexList.add(new Integer(index)); + + index++; + } + } + parsedQuery.append(c); + } + + return new PreparsedQuery(parsedQuery.toString(), paramMap); + } + + // private helper classes + + private static class PreparsedQuery { + private final Map<String, List<Integer>> indexMap; + private final String sql; + + public PreparsedQuery(String sql, Map<String, List<Integer>> indexMap) { + this.sql = sql; + this.indexMap = indexMap; + } + } + + private static class QueryCache extends LinkedHashMap<String, PreparsedQuery> { + private static final long serialVersionUID = 1L; + + public QueryCache() { + super(30, (float) 0.75, true); + } + + @Override + protected boolean removeEldestEntry(Map.Entry<String, PreparsedQuery> eldest) { + return size() > 40; + } + } + +} diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbImage.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbImage.java new file mode 100644 index 00000000..b772edb4 --- /dev/null +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbImage.java @@ -0,0 +1,62 @@ +package org.openslx.bwlp.sat.database.mappers; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.log4j.Logger; +import org.openslx.bwlp.sat.database.Database; +import org.openslx.bwlp.sat.database.MysqlConnection; +import org.openslx.bwlp.sat.database.MysqlStatement; +import org.openslx.bwlp.thrift.iface.ImagePermissions; +import org.openslx.bwlp.thrift.iface.ImageSummaryRead; +import org.openslx.bwlp.thrift.iface.ShareMode; +import org.openslx.bwlp.thrift.iface.UserInfo; + +public class DbImage { + + private static final Logger LOGGER = Logger.getLogger(DbImage.class); + + public static List<ImageSummaryRead> getAllVisible(UserInfo user, List<String> tagSearch) { + try (MysqlConnection connection = Database.getConnection()) { + MysqlStatement stmt = connection.prepareStatement("SELECT" + + " i.imagebaseid, i.currentversionid, i.latestversionid, i.displayname," + + " i.osid, i.virtid, i.createtime, i.updatetime, i.ownerid," + + " i.sharemode, i.istemplate, i.canlinkdefault, i.candownloaddefault," + + " i.caneditdefault, i.canadmindefault," + + " cur.expiretime, cur.filesize, cur.isenabled, cur.isrestricted, cur.isvalid," + + " lat.uploaderid, lat.isprocessed," + + " perm.canlink, perm.candownload, perm.canedit, perm.canadmin" + + " FROM imagebase i" + + " LEFT JOIN imageversion cur ON (cur.imageversionid = i.currentversionid)" + + " LEFT JOIN imageversion lat ON (lat.imageversionid = i.latestversionid)" + + " LEFT JOIN imagepermission perm ON (i.imagebaseid = perm.imagebaseid AND perm.userid = :userid)"); + stmt.setString("userid", user.userId); + ResultSet rs = stmt.executeQuery(); + List<ImageSummaryRead> list = new ArrayList<>(); + while (rs.next()) { + ImagePermissions defaultPermissions = DbImagePermissions.fromResultSetDefault(rs); + ImageSummaryRead entry = new ImageSummaryRead(rs.getString("imagebaseid"), + rs.getString("currentversionid"), rs.getString("latestversionid"), + rs.getString("displayname"), rs.getInt("osid"), rs.getString("virtid"), + rs.getLong("createtime"), rs.getLong("updatetime"), rs.getLong("expiretime"), + rs.getString("ownerid"), rs.getString("uploaderid"), + toShareMode(rs.getString("sharemode")), rs.getLong("filesize"), + rs.getByte("isrestricted") != 0, rs.getByte("isvalid") != 0, + rs.getByte("isprocessed") != 0, rs.getByte("istemplate") != 0, defaultPermissions); + entry.userPermissions = DbImagePermissions.fromResultSetUser(rs); + list.add(entry); + } + return list; + } catch (SQLException e) { + LOGGER.error("Query failed in DbImage.getAllVisible()", e); + return null; + } + } + + private static ShareMode toShareMode(String string) { + return ShareMode.valueOf(string); + } + +} diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbImagePermissions.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbImagePermissions.java new file mode 100644 index 00000000..e254b085 --- /dev/null +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbImagePermissions.java @@ -0,0 +1,64 @@ +package org.openslx.bwlp.sat.database.mappers; + +import java.sql.ResultSet; +import java.sql.SQLException; + +import org.openslx.bwlp.thrift.iface.ImagePermissions; + +public class DbImagePermissions { + + /** + * Build an instance of {@link ImagePermissions} by reading the given + * columns from the given {@link ResultSet}. If there are no permissions + * given in the ResultSet, <code>null</code> is returned. + * + * @param rs the {@link ResultSet} to read from + * @param canLink Name of the column to read the "can link" permission from + * @param canDownload Name of the column to read the "can download" + * permission from + * @param canEdit Name of the column to read the "can edit" permission from + * @param canAdmin Name of the column to read the "can admin" permission + * from + * @return instance of {@link ImagePermissions}, or <code>null</code> + * @throws SQLException + */ + private static ImagePermissions fromResultSet(ResultSet rs, String canLink, String canDownload, + String canEdit, String canAdmin) throws SQLException { + byte link = rs.getByte(canLink); + if (rs.wasNull()) + return null; + return new ImagePermissions(link != 0, rs.getByte(canDownload) != 0, rs.getByte(canEdit) != 0, + rs.getByte(canAdmin) != 0); + } + + /** + * Build an instance of {@link ImagePermissions} by reading the + * columns <code>canlink</code>, <code>candownload</code>, + * <code>canedit</code>, <code>canadmin</code> from the given + * {@link ResultSet}. If there are no permissions + * given in the ResultSet, <code>null</code> is returned. + * + * @param rs the {@link ResultSet} to read from + * @return instance of {@link ImagePermissions}, or <code>null</code> + * @throws SQLException + */ + public static ImagePermissions fromResultSetUser(ResultSet rs) throws SQLException { + return fromResultSet(rs, "canlink", "candownload", "canedit", "canadmin"); + } + + /** + * Build an instance of {@link ImagePermissions} by reading the + * columns <code>canlinkdefault</code>, <code>candownloaddefault</code>, + * <code>caneditdefault</code>, <code>canadmindefault</code> from the given + * {@link ResultSet}. If there are no permissions + * given in the ResultSet, <code>null</code> is returned. + * + * @param rs the {@link ResultSet} to read from + * @return instance of {@link ImagePermissions}, or <code>null</code> + * @throws SQLException + */ + public static ImagePermissions fromResultSetDefault(ResultSet rs) throws SQLException { + return fromResultSet(rs, "canlinkdefault", "candownloaddefault", "caneditdefault", "canadmindefault"); + } + +} diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbOrganization.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbOrganization.java new file mode 100644 index 00000000..cc401af9 --- /dev/null +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbOrganization.java @@ -0,0 +1,34 @@ +package org.openslx.bwlp.sat.database.mappers; + +import java.sql.SQLException; +import java.util.List; + +import org.apache.log4j.Logger; +import org.openslx.bwlp.sat.database.Database; +import org.openslx.bwlp.sat.database.MysqlConnection; +import org.openslx.bwlp.sat.database.MysqlStatement; +import org.openslx.bwlp.thrift.iface.Organization; + +public class DbOrganization { + + private static final Logger LOGGER = Logger.getLogger(DbOrganization.class); + + public static boolean storeOrganizations(List<Organization> organizations) { + try (MysqlConnection connection = Database.getConnection()) { + MysqlStatement stmt = connection.prepareStatement("INSERT INTO organization" + + " (organizationid, displayname, canlogin) VALUES (:id, :name, 0)" + + " ON DUPLICATE KEY UPDATE displayname = VALUES(displayname)"); + for (Organization organization : organizations) { + stmt.setString("id", organization.organizationId); + stmt.setString("name", organization.displayName); + stmt.executeUpdate(); + } + connection.commit(); + return true; + } catch (SQLException e) { + LOGGER.error("Query failed in DbOrganization.storeOrganization()", e); + return false; + } + } + +} diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/fileserv/ActiveUpload.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/fileserv/ActiveUpload.java new file mode 100644 index 00000000..a2474587 --- /dev/null +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/fileserv/ActiveUpload.java @@ -0,0 +1,207 @@ +package org.openslx.bwlp.sat.fileserv; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.ThreadPoolExecutor; + +import org.apache.log4j.Logger; +import org.openslx.bwlp.sat.util.Configuration; +import org.openslx.bwlp.sat.util.FileSystem; +import org.openslx.bwlp.sat.util.Formatter; +import org.openslx.bwlp.thrift.iface.ImageDetailsRead; +import org.openslx.bwlp.thrift.iface.UserInfo; +import org.openslx.filetransfer.DataReceivedCallback; +import org.openslx.filetransfer.Downloader; +import org.openslx.filetransfer.FileRange; +import org.openslx.filetransfer.WantRangeCallback; + +public class ActiveUpload { + private static final Logger LOGGER = Logger.getLogger(ActiveUpload.class); + + /** + * This is an active upload, so on our end, we have a Downloader. + */ + private Downloader download = null; + + private final File destinationFile; + + private final RandomAccessFile outFile; + + private final ChunkList chunks; + + private final long fileSize; + + /** + * User owning this uploaded file. + */ + private final UserInfo owner; + + /** + * Base image this upload is a new version for. + */ + private final ImageDetailsRead image; + + // TODO: Use HashList for verification + + public ActiveUpload(UserInfo owner, ImageDetailsRead image, File destinationFile, long fileSize, + List<ByteBuffer> sha1Sums) throws FileNotFoundException { + this.destinationFile = destinationFile; + this.outFile = new RandomAccessFile(destinationFile, "rw"); + this.chunks = new ChunkList(fileSize, sha1Sums); + this.owner = owner; + this.image = image; + this.fileSize = fileSize; + } + + /** + * Add another connection for this file transfer. Currently only one + * connection is allowed, but this might change in the future. + * + * @param connection + * @return true if the connection is accepted, false if it should be + * discarded + */ + public synchronized boolean addConnection(Downloader connection, ThreadPoolExecutor pool) { + if (download != null || chunks.isComplete()) + return false; + download = connection; + pool.execute(new Runnable() { + @Override + public void run() { + CbHandler cbh = new CbHandler(); + if (!download.download(cbh, cbh) && cbh.currentChunk != null) { + // If the download failed and we have a current chunk, put it back into + // the queue, so it will be handled again later... + chunks.markFailed(cbh.currentChunk); + } + } + }); + return true; + } + + /** + * Write some data to the local file. Thread safe so we could + * have multiple concurrent connections later. + * + * @param fileOffset + * @param dataLength + * @param data + * @return + */ + private boolean writeFileData(long fileOffset, int dataLength, byte[] data) { + synchronized (outFile) { + try { + outFile.seek(fileOffset); + outFile.write(data, 0, dataLength); + } catch (IOException e) { + LOGGER.error("Cannot write to '" + destinationFile + + "'. Disk full, network storage error, bad permissions, ...?", e); + return false; + } + } + return true; + } + + private void finishUpload() { + File file = destinationFile; + // Ready to go. First step: Rename temp file to something usable + File destination = new File(file.getParent(), Formatter.vmName(owner, image.imageName)); + // Sanity check: destination should be a sub directory of the vmStorePath + String relPath = FileSystem.getRelativePath(destination, Configuration.getVmStoreBasePath()); + if (relPath == null) { + LOGGER.warn(destination.getAbsolutePath() + " is not a subdir of " + + Configuration.getVmStoreBasePath().getAbsolutePath()); + // TODO: Update state to failed... + } + + // Execute rename + boolean ret = false; + Exception renameException = null; + try { + ret = file.renameTo(destination); + } catch (Exception e) { + ret = false; + renameException = e; + } + if (!ret) { + // Rename failed :-( + LOGGER.warn( + "Could not rename '" + file.getAbsolutePath() + "' to '" + destination.getAbsolutePath() + + "'", renameException); + // TODO: Update state.... + } + + // Now insert meta data into DB + + final String imageVersionId = UUID.randomUUID().toString(); + + // TODO: SQL magic, update state + } + + public void cancel() { + // TODO Auto-generated method stub + + } + + /** + * Get user owning this upload. Can be null in special cases. + * + * @return instance of UserInfo for the according user. + */ + public UserInfo getOwner() { + return this.owner; + } + + public boolean isComplete() { + return chunks.isComplete() && destinationFile.length() == this.fileSize; + } + + public File getDestinationFile() { + return this.destinationFile; + } + + public long getSize() { + return this.fileSize; + } + + /** + * Callback class for an instance of the Downloader, which supplies + * the Downloader with wanted file ranges, and handles incoming data. + */ + private class CbHandler implements WantRangeCallback, DataReceivedCallback { + /** + * The current chunk being transfered. + */ + public FileChunk currentChunk = null; + + @Override + public boolean dataReceived(long fileOffset, int dataLength, byte[] data) { + // TODO: Maybe cache in RAM and write full CHUNK_SIZE blocks at a time? + // Would probably help with slower storage, especially if it's using + // rotating disks and we're running multiple uploads. + // Also we wouldn't have to re-read a block form disk for sha1 checking. + return writeFileData(fileOffset, dataLength, data); + } + + @Override + public FileRange get() { + if (currentChunk != null) { + // TODO: A chunk was requested before, check hash and requeue if not matching + // This needs to be async (own thread) so will be a little complicated + } + // Get next missing chunk + currentChunk = chunks.getMissing(); + if (currentChunk == null) + return null; // No more chunks, returning null tells the Downloader we're done. + return currentChunk.range; + } + } + + // TODO: Clean up old stale uploads + +} diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/fileserv/ChunkList.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/fileserv/ChunkList.java new file mode 100644 index 00000000..b07193c5 --- /dev/null +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/fileserv/ChunkList.java @@ -0,0 +1,78 @@ +package org.openslx.bwlp.sat.fileserv; + +import java.nio.ByteBuffer; +import java.util.LinkedList; +import java.util.List; + +import org.apache.log4j.Logger; + +public class ChunkList { + + private static final Logger LOGGER = Logger.getLogger(ChunkList.class); + + /** + * Chunks that are missing from the file + */ + private final List<FileChunk> missingChunks = new LinkedList<>(); + + /** + * Chunks that are currently being uploaded or hash-checked + */ + private final List<FileChunk> pendingChunks = new LinkedList<>(); + + // Do we need to keep valid chunks, or chunks that failed too many times? + + public ChunkList(long fileSize, List<ByteBuffer> sha1Sums) { + FileChunk.createChunkList(missingChunks, fileSize, sha1Sums); + } + + /** + * Get a missing chunk, marking it pending. + * + * @return chunk marked as missing + */ + public synchronized FileChunk getMissing() { + if (missingChunks.isEmpty()) + return null; + FileChunk c = missingChunks.remove(0); + pendingChunks.add(c); + return c; + } + + /** + * Mark a chunk currently transferring as successfully transfered. + * + * @param c The chunk in question + */ + public synchronized void markSuccessful(FileChunk c) { + if (!pendingChunks.remove(c)) { + LOGGER.warn("Inconsistent state: markTransferred called for Chunk " + c.toString() + + ", but chunk is not marked as currently transferring!"); + return; + } + } + + /** + * Mark a chunk currently transferring or being hash checked as failed + * transfer. This increases its fail count and re-adds it to the list of + * missing chunks. + * + * @param c The chunk in question + * @return Number of times transfer of this chunk failed + */ + public synchronized int markFailed(FileChunk c) { + if (!pendingChunks.remove(c)) { + LOGGER.warn("Inconsistent state: markTransferred called for Chunk " + c.toString() + + ", but chunk is not marked as currently transferring!"); + return -1; + } + // Add as first element so it will be re-transmitted immediately + missingChunks.add(0, c); + return c.incFailed(); + } + + public synchronized boolean isComplete() { + return missingChunks.isEmpty() && pendingChunks.isEmpty(); + } + +} diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/fileserv/FileChunk.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/fileserv/FileChunk.java new file mode 100644 index 00000000..ffa033a5 --- /dev/null +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/fileserv/FileChunk.java @@ -0,0 +1,66 @@ +package org.openslx.bwlp.sat.fileserv; + +import java.nio.ByteBuffer; +import java.util.Collection; +import java.util.List; + +import org.openslx.filetransfer.FileRange; + +public class FileChunk { + + public static final int CHUNK_SIZE_MIB = 16; + public static final int CHUNK_SIZE = CHUNK_SIZE_MIB * (1024 * 1024); + + public final FileRange range; + public final byte[] sha1sum; + private int failCount = 0; + + public FileChunk(long startOffset, long endOffset, byte[] sha1sum) { + this.range = new FileRange(startOffset, endOffset); + this.sha1sum = sha1sum; + } + + /** + * Signal that transferring this chunk seems to have failed (checksum + * mismatch). + * + * @return Number of times the transfer failed now + */ + public synchronized int incFailed() { + return ++failCount; + } + + // + + public static int fileSizeToChunkCount(long fileSize) { + return (int) ((fileSize + CHUNK_SIZE - 1) / CHUNK_SIZE); + } + + public static void createChunkList(Collection<FileChunk> list, long fileSize, List<ByteBuffer> sha1Sums) { + if (fileSize < 0) + throw new IllegalArgumentException("fileSize cannot be negative"); + long chunkCount = fileSizeToChunkCount(fileSize); + if (sha1Sums != null) { + if (sha1Sums.size() != chunkCount) + throw new IllegalArgumentException( + "Passed a sha1sum list, but hash count in list doesn't match expected chunk count"); + long offset = 0; + for (ByteBuffer sha1sum : sha1Sums) { // Do this as we don't know how efficient List.get(index) is... + long end = offset + CHUNK_SIZE; + if (end > fileSize) + end = fileSize; + list.add(new FileChunk(offset, end, sha1sum.array())); + offset = end; + } + return; + } + long offset = 0; + while (offset < fileSize) { // ...otherwise we could share this code + long end = offset + CHUNK_SIZE; + if (end > fileSize) + end = fileSize; + list.add(new FileChunk(offset, end, null)); + offset = end; + } + } +} diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/fileserv/FileServer.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/fileserv/FileServer.java new file mode 100644 index 00000000..c357c292 --- /dev/null +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/fileserv/FileServer.java @@ -0,0 +1,106 @@ +package org.openslx.bwlp.sat.fileserv; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; + +import org.openslx.bwlp.sat.util.Constants; +import org.openslx.bwlp.sat.util.Formatter; +import org.openslx.bwlp.thrift.iface.TTransferRejectedException; +import org.openslx.bwlp.thrift.iface.UserInfo; +import org.openslx.filetransfer.Downloader; +import org.openslx.filetransfer.IncomingEvent; +import org.openslx.filetransfer.Listener; +import org.openslx.filetransfer.Uploader; + +public class FileServer implements IncomingEvent { + + /** + * Listener for incoming unencrypted connections + */ + private Listener plainListener = new Listener(this, null, 9092); // TODO: Config + + /** + * All currently running uploads, indexed by token + */ + private Map<String, ActiveUpload> uploads = new ConcurrentHashMap<>(); + + private static final FileServer globalInstance = new FileServer(); + + private FileServer() { + } + + public static FileServer instance() { + return globalInstance; + } + + public boolean start() { + boolean ret = plainListener.start(); + // TODO: Start SSL listener too + return ret; + } + + @Override + public void incomingDownloadRequest(Uploader uploader) throws IOException { + // TODO Auto-generated method stub + + } + + @Override + public void incomingUploadRequest(Downloader downloader) throws IOException { + // TODO Auto-generated method stub + + } + + /** + * Get an upload instance by given token. + * + * @param uploadToken + * @return + */ + public ActiveUpload getUploadByToken(String uploadToken) { + return uploads.get(uploadToken); + } + + public String createNewUserUpload(UserInfo owner, long fileSize, List<ByteBuffer> sha1Sums) + throws TTransferRejectedException, FileNotFoundException { + Iterator<ActiveUpload> it = uploads.values().iterator(); + int activeUploads = 0; + while (it.hasNext()) { + ActiveUpload upload = it.next(); + if (upload.isComplete()) { + // TODO: Check age (short timeout) and remove + continue; + } else { + // Check age (long timeout) and remove + } + activeUploads++; + } + if (activeUploads > Constants.MAX_UPLOADS) + throw new TTransferRejectedException("Server busy. Too many running uploads."); + File destinationFile = null; + do { + destinationFile = Formatter.getTempImageName(); + } while (destinationFile.exists()); + // TODO: Pass image + ActiveUpload upload = new ActiveUpload(owner, null, destinationFile, fileSize, sha1Sums); + String key = UUID.randomUUID().toString(); + uploads.put(key, upload); + return key; + } + + public int getPlainPort() { + return plainListener.getPort(); + } + + public int getSslPort() { + return 0; // TODO + } + +} diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/thrift/BinaryListener.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/thrift/BinaryListener.java new file mode 100644 index 00000000..70c47edb --- /dev/null +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/thrift/BinaryListener.java @@ -0,0 +1,65 @@ +package org.openslx.bwlp.sat.thrift; + +import java.security.NoSuchAlgorithmException; + +import org.apache.log4j.Logger; +import org.apache.thrift.protocol.TProtocolFactory; +import org.apache.thrift.server.THsHaServer; +import org.apache.thrift.server.TServer; +import org.apache.thrift.transport.TNonblockingServerSocket; +import org.apache.thrift.transport.TNonblockingServerTransport; +import org.apache.thrift.transport.TTransportException; +import org.openslx.bwlp.thrift.iface.SatelliteServer; +import org.openslx.thrifthelper.TBinaryProtocolSafe; + +public class BinaryListener implements Runnable { + private static final Logger log = Logger.getLogger(BinaryListener.class); + + private static final int MAX_MSG_LEN = 30 * 1000 * 1000; + private static final int MINWORKERTHREADS = 2; + private static final int MAXWORKERTHREADS = 64; + + private final SatelliteServer.Processor<ServerHandler> processor = new SatelliteServer.Processor<ServerHandler>( + new ServerHandler()); + private final TProtocolFactory protFactory = new TBinaryProtocolSafe.Factory(true, true); + + private final TServer server; + + public BinaryListener(int port, boolean secure) throws TTransportException, NoSuchAlgorithmException { + if (secure) + server = initSecure(port); + else + server = initNormal(port); + } + + @Override + public void run() { + log.info("Starting Listener"); + server.serve(); + log.fatal("Listener stopped unexpectedly"); + // TODO: Restart listener; if it fails, quit server so it will be restarted by the OS + } + + private TServer initSecure(int port) throws NoSuchAlgorithmException, TTransportException { + // TODO + return null; + } + + private TServer initNormal(int port) throws TTransportException { + final TNonblockingServerTransport serverTransport; + try { + serverTransport = new TNonblockingServerSocket(port); + log.fatal("Listening on port " + port + " (plain handler)"); + } catch (TTransportException e) { + log.fatal("Could not listen on port " + port + " (plain handler)"); + throw e; + } + THsHaServer.Args args = new THsHaServer.Args(serverTransport); + args.protocolFactory(protFactory); + args.processor(processor); + args.workerThreads(8); + args.maxReadBufferBytes = MAX_MSG_LEN; + return new THsHaServer(args); + } + +} diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/thrift/ServerHandler.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/thrift/ServerHandler.java new file mode 100644 index 00000000..cf26b510 --- /dev/null +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/thrift/ServerHandler.java @@ -0,0 +1,214 @@ +package org.openslx.bwlp.sat.thrift; + +import java.nio.ByteBuffer; +import java.util.List; +import java.util.Map; + +import org.apache.log4j.Logger; +import org.apache.thrift.TException; +import org.openslx.bwlp.sat.database.mappers.DbImage; +import org.openslx.bwlp.sat.fileserv.ActiveUpload; +import org.openslx.bwlp.sat.fileserv.FileServer; +import org.openslx.bwlp.sat.thrift.cache.OperatingSystemList; +import org.openslx.bwlp.thrift.iface.ImageBaseWrite; +import org.openslx.bwlp.thrift.iface.ImageDetailsRead; +import org.openslx.bwlp.thrift.iface.ImagePermissions; +import org.openslx.bwlp.thrift.iface.ImageSummaryRead; +import org.openslx.bwlp.thrift.iface.ImageVersionWrite; +import org.openslx.bwlp.thrift.iface.LecturePermissions; +import org.openslx.bwlp.thrift.iface.LectureRead; +import org.openslx.bwlp.thrift.iface.LectureSummary; +import org.openslx.bwlp.thrift.iface.LectureWrite; +import org.openslx.bwlp.thrift.iface.OperatingSystem; +import org.openslx.bwlp.thrift.iface.Organization; +import org.openslx.bwlp.thrift.iface.SatelliteServer; +import org.openslx.bwlp.thrift.iface.TAuthorizationException; +import org.openslx.bwlp.thrift.iface.TInvalidTokenException; +import org.openslx.bwlp.thrift.iface.TNotFoundException; +import org.openslx.bwlp.thrift.iface.TTransferRejectedException; +import org.openslx.bwlp.thrift.iface.TransferInformation; +import org.openslx.bwlp.thrift.iface.UploadStatus; +import org.openslx.bwlp.thrift.iface.UserInfo; +import org.openslx.bwlp.thrift.iface.Virtualizer; +import org.openslx.sat.thrift.version.Version; + +public class ServerHandler implements SatelliteServer.Iface { + + private static final Logger log = Logger.getLogger(ServerHandler.class); + + private static final FileServer fileServer = FileServer.instance(); + + @Override + public long getVersion() throws TException { + return Version.VERSION; + } + + @Override + public TransferInformation requestImageVersionUpload(String userToken, String imageBaseId, long fileSize, + List<ByteBuffer> blockHashes) throws TTransferRejectedException, TAuthorizationException, + TException { + // TODO Auto-generated method stub + return null; + } + + @Override + public void cancelUpload(String uploadToken) throws TException { + ActiveUpload upload = fileServer.getUploadByToken(uploadToken); + if (upload != null) + upload.cancel(); + + } + + @Override + public UploadStatus queryUploadStatus(String uploadToken) throws TInvalidTokenException, TException { + // TODO Auto-generated method stub + return null; + } + + @Override + public TransferInformation requestDownload(String userToken, String imageVersionId) + throws TAuthorizationException, TException { + // TODO Auto-generated method stub + return null; + } + + @Override + public void cancelDownload(String downloadToken) throws TException { + // TODO Auto-generated method stub + + } + + @Override + public boolean isAuthenticated(String userToken) throws TException { + return SessionManager.get(userToken) != null; + } + + @Override + public void invalidateSession(String userToken) throws TException { + SessionManager.remove(userToken); + } + + @Override + public List<OperatingSystem> getOperatingSystems() throws TException { + return OperatingSystemList.get(); + } + + @Override + public List<Virtualizer> getVirtualizers() throws TException { + // TODO Auto-generated method stub + return null; + } + + @Override + public List<Organization> getAllOrganizations() throws TException { + // TODO Auto-generated method stub + return null; + } + + @Override + public List<ImageSummaryRead> getImageList(String userToken, List<String> tagSearch) + throws TAuthorizationException, TException { + UserInfo user = SessionManager.getOrFail(userToken); + return DbImage.getAllVisible(user, tagSearch); + } + + @Override + public ImageDetailsRead getImageDetails(String userToken, String imageBaseId) + throws TAuthorizationException, TNotFoundException, TException { + // TODO Auto-generated method stub + return null; + } + + @Override + public boolean updateImageBase(String userToken, String imageBaseId, ImageBaseWrite image) + throws TAuthorizationException, TException { + // TODO Auto-generated method stub + return false; + } + + @Override + public boolean updateImageVersion(String userToken, String imageVersionId, ImageVersionWrite image) + throws TAuthorizationException, TException { + // TODO Auto-generated method stub + return false; + } + + @Override + public boolean deleteImageVersion(String userToken, String imageVersionId) + throws TAuthorizationException, TNotFoundException, TException { + // TODO Auto-generated method stub + return false; + } + + @Override + public boolean writeImagePermissions(String userToken, String imageId, + Map<String, ImagePermissions> permissions) throws TAuthorizationException, TNotFoundException, + TException { + // TODO Auto-generated method stub + return false; + } + + @Override + public Map<String, ImagePermissions> getImagePermissions(String userToken, String imageBaseId) + throws TAuthorizationException, TNotFoundException, TException { + // TODO Auto-generated method stub + return null; + } + + @Override + public String createLecture(String userToken, LectureWrite lecture) throws TAuthorizationException, + TException { + // TODO Auto-generated method stub + return null; + } + + @Override + public boolean updateLecture(String userToken, String lectureId, LectureWrite lecture) + throws TAuthorizationException, TNotFoundException, TException { + // TODO Auto-generated method stub + return false; + } + + @Override + public List<LectureSummary> getLectureList(String userToken) throws TAuthorizationException, TException { + // TODO Auto-generated method stub + return null; + } + + @Override + public LectureRead getLectureDetails(String userToken, String lectureId) throws TAuthorizationException, + TNotFoundException, TException { + // TODO Auto-generated method stub + return null; + } + + @Override + public List<LectureSummary> getLecturesByImageVersion(String userToken, String imageVersionId) + throws TAuthorizationException, TNotFoundException, TException { + // TODO Auto-generated method stub + return null; + } + + @Override + public boolean deleteLecture(String userToken, String lectureId) throws TAuthorizationException, + TNotFoundException, TException { + // TODO Auto-generated method stub + return false; + } + + @Override + public boolean writeLecturePermissions(String userToken, String lectureId, + Map<String, LecturePermissions> permissions) throws TAuthorizationException, TNotFoundException, + TException { + // TODO Auto-generated method stub + return false; + } + + @Override + public Map<String, LecturePermissions> getLecturePermissions(String userToken, String lectureId) + throws TAuthorizationException, TNotFoundException, TException { + // TODO Auto-generated method stub + return null; + } + +}// end class diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/thrift/SessionManager.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/thrift/SessionManager.java new file mode 100644 index 00000000..bf444a20 --- /dev/null +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/thrift/SessionManager.java @@ -0,0 +1,101 @@ +package org.openslx.bwlp.sat.thrift; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; + +import org.apache.log4j.Logger; +import org.openslx.bwlp.thrift.iface.AuthorizationError; +import org.openslx.bwlp.thrift.iface.TAuthorizationException; +import org.openslx.bwlp.thrift.iface.UserInfo; +import org.openslx.thrifthelper.ThriftManager; + +/** + * Manages user sessions. Mainly used to map tokens to users. + * + */ +public class SessionManager { + + private static final Logger LOGGER = Logger.getLogger(SessionManager.class); + + private static class Entry { + private static final long SESSION_TIMEOUT = TimeUnit.DAYS.toMillis(1); + private final UserInfo user; + private long validUntil; + + private Entry(UserInfo user) { + this.user = user; + this.validUntil = System.currentTimeMillis() + SESSION_TIMEOUT; + } + + public void touch(long now) { + this.validUntil = now + SESSION_TIMEOUT; + } + } + + // saves the current tokens and the mapped userdata, returning from the server + private static Map<String, Entry> tokenManager = new ConcurrentHashMap<>(); + + /** + * Get the user corresponding to the given token. + * + * @param token user's token + * @return UserInfo for the matching user + * @throws TAuthorizationException if the token is not known or the session + * expired + */ + public static UserInfo getOrFail(String token) throws TAuthorizationException { + UserInfo ui = get(token); + if (ui != null) + return ui; + throw new TAuthorizationException(AuthorizationError.NOT_AUTHENTICATED, + "Your session token is not known to the server"); + } + + /** + * Get the user corresponding to the given token. Returns null if the token + * is not known, or the session already timed out. + * + * @param token user's token + * @return UserInfo for the matching user + */ + public static UserInfo get(String token) { + Entry e = tokenManager.get(token); + if (e != null) { + // User session already cached + final long now = System.currentTimeMillis(); + if (e.validUntil < now) { + tokenManager.remove(token); + return getRemote(token); + } + e.touch(now); + return e.user; + } + return getRemote(token); + } + + /** + * Remove session matching the given token + * + * @param token + */ + public static void remove(String token) { + tokenManager.remove(token); + } + + private static UserInfo getRemote(String token) { + UserInfo ui = null; + try { + ui = ThriftManager.getMasterClient().getUserFromToken(token); + } catch (Exception e) { + LOGGER.warn("Could not reach master server to query for user token of a client!", e); + } + if (ui == null) + return null; + tokenManager.put(token, new Entry(ui)); + return ui; + } + + // TODO: Clean map of old entries periodically + +} diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/thrift/cache/CachedList.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/thrift/cache/CachedList.java new file mode 100644 index 00000000..4c986fd2 --- /dev/null +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/thrift/cache/CachedList.java @@ -0,0 +1,33 @@ +package org.openslx.bwlp.sat.thrift.cache; + +import java.util.List; + +import org.apache.log4j.Logger; +import org.apache.thrift.TException; +import org.openslx.util.TimeoutReference; + + +public abstract class CachedList<T> { + + private static final Logger LOGGER = Logger.getLogger(CachedList.class); + + private final TimeoutReference<List<T>> cachedList = new TimeoutReference<>(600000, null); + + protected abstract List<T> getCallback() throws TException; + + protected synchronized List<T> getInternal() { + List<T> list = cachedList.get(); + if (list == null) { + try { + list = getCallback(); + } catch (TException e) { + LOGGER.warn("Could not retrieve " + getClass().getSimpleName() + " list from master server", + e); + return null; + } + cachedList.set(list); + } + return list; + } + +} diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/thrift/cache/OperatingSystemList.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/thrift/cache/OperatingSystemList.java new file mode 100644 index 00000000..020ae4ff --- /dev/null +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/thrift/cache/OperatingSystemList.java @@ -0,0 +1,26 @@ +package org.openslx.bwlp.sat.thrift.cache; + +import java.util.List; + +import org.apache.thrift.TException; +import org.openslx.bwlp.thrift.iface.OperatingSystem; +import org.openslx.thrifthelper.ThriftManager; + +/** + * Holds the list of all known organizations. The list is synchronized with + * the master server. + */ +public class OperatingSystemList extends CachedList<OperatingSystem> { + + private static final OperatingSystemList instance = new OperatingSystemList(); + + public static List<OperatingSystem> get() { + return instance.getInternal(); + } + + @Override + protected List<OperatingSystem> getCallback() throws TException { + return ThriftManager.getMasterClient().getOperatingSystems(); + } + +} diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/thrift/cache/OrganizationList.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/thrift/cache/OrganizationList.java new file mode 100644 index 00000000..8db7e7e5 --- /dev/null +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/thrift/cache/OrganizationList.java @@ -0,0 +1,29 @@ +package org.openslx.bwlp.sat.thrift.cache; + +import java.util.List; + +import org.apache.thrift.TException; +import org.openslx.bwlp.sat.database.mappers.DbOrganization; +import org.openslx.bwlp.thrift.iface.Organization; +import org.openslx.thrifthelper.ThriftManager; + +/** + * Holds the list of all known organizations. The list is synchronized with + * the master server. + */ +public class OrganizationList extends CachedList<Organization> { + + private static final OrganizationList instance = new OrganizationList(); + + public static List<Organization> get() { + return instance.getInternal(); + } + + @Override + protected List<Organization> getCallback() throws TException { + List<Organization> organizations = ThriftManager.getMasterClient().getOrganizations(); + DbOrganization.storeOrganizations(organizations); + return organizations; + } + +} diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/util/Configuration.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/util/Configuration.java new file mode 100644 index 00000000..07cd3a8d --- /dev/null +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/util/Configuration.java @@ -0,0 +1,72 @@ +package org.openslx.bwlp.sat.util; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Properties; + +import org.apache.log4j.Logger; +import org.joda.time.format.DateTimeFormat; +import org.joda.time.format.DateTimeFormatter; + +public class Configuration { + + private static final Logger LOGGER = Logger.getLogger(Configuration.class); + private static final DateTimeFormatter subdirDate = DateTimeFormat.forPattern("yy-MM"); + + private static File vmStoreBasePath; + private static File vmStoreProdPath; + private static String dbUri; + private static String dbUsername; + private static String dbPassword; + + public static boolean load() throws IOException { + // Load configuration from java properties file + Properties prop = new Properties(); + InputStream in = new FileInputStream("./config.properties"); + try { + prop.load(in); + } finally { + in.close(); + } + + vmStoreBasePath = new File(prop.getProperty("vmstore.path")); + vmStoreProdPath = new File(vmStoreBasePath, "prod"); + dbUri = prop.getProperty("db.uri"); + dbUsername = prop.getProperty("db.username"); + dbPassword = prop.getProperty("db.password"); + + // Currently all fields are mandatory but there might be optional settings in the future + return vmStoreBasePath != null && dbUri != null && dbUsername != null && dbPassword != null; + } + + // Static ("real") fields + + public static File getVmStoreBasePath() { + return vmStoreBasePath; + } + + public static String getDbUri() { + return dbUri; + } + + public static String getDbUsername() { + return dbUsername; + } + + public static String getDbPassword() { + return dbPassword; + } + + public static File getVmStoreProdPath() { + return vmStoreProdPath; + } + + // Dynamically Computed fields + + public static File getCurrentVmStorePath() { + return new File(vmStoreProdPath, subdirDate.print(System.currentTimeMillis())); + } + +} diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/util/Constants.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/util/Constants.java new file mode 100644 index 00000000..6c2dc31b --- /dev/null +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/util/Constants.java @@ -0,0 +1,21 @@ +package org.openslx.bwlp.sat.util; + +import org.openslx.bwlp.sat.fileserv.FileChunk; + +public class Constants { + public static final String INCOMPLETE_UPLOAD_SUFFIX = ".part"; + public static final int MAX_UPLOADS; + + static { + long maxMem = Runtime.getRuntime().maxMemory(); + if (maxMem == Long.MAX_VALUE) { + // Apparently the JVM was started without a memory limit (no -Xmx cmdline), + // so we assume a default of 256MB + maxMem = 256l * 1024l * 1024l; + } + maxMem /= 1024l * 1024l; + // Now maxMem is the amount of memory in MiB + + MAX_UPLOADS = (int) Math.max(1, (maxMem - 64) / (FileChunk.CHUNK_SIZE_MIB + 1)); + } +} diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/util/FileSystem.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/util/FileSystem.java new file mode 100644 index 00000000..38841cd9 --- /dev/null +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/util/FileSystem.java @@ -0,0 +1,25 @@ +package org.openslx.bwlp.sat.util; + +import java.io.File; + +import org.apache.log4j.Logger; + +public class FileSystem { + + private static final Logger LOGGER = Logger.getLogger(FileSystem.class); + + public static String getRelativePath(File absolutePath, File parentDir) { + String file; + String dir; + try { + file = absolutePath.getCanonicalPath(); + dir = parentDir.getCanonicalPath() + File.separator; + } catch (Exception e) { + LOGGER.error("Could not get relative path for " + absolutePath.toString(), e); + return null; + } + if (!file.startsWith(dir)) + return null; + return file.substring(dir.length()); + } +} diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/util/Formatter.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/util/Formatter.java new file mode 100644 index 00000000..0839ad24 --- /dev/null +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/util/Formatter.java @@ -0,0 +1,57 @@ +package org.openslx.bwlp.sat.util; + +import java.io.File; +import java.util.UUID; + +import org.joda.time.format.DateTimeFormat; +import org.joda.time.format.DateTimeFormatter; +import org.openslx.bwlp.thrift.iface.UserInfo; + +public class Formatter { + + private static final DateTimeFormatter vmNameDateFormat = DateTimeFormat.forPattern("dd_HH-mm-ss"); + + /** + * Generate a unique file name used for a virtual machine + * image that is currently uploading. + * + * @return Absolute path name of file + */ + public static File getTempImageName() { + return new File(Configuration.getCurrentVmStorePath(), UUID.randomUUID().toString() + + Constants.INCOMPLETE_UPLOAD_SUFFIX); + } + + /** + * Generate a file name for the given VM based on owner and display name. + * + * @param user The user associated with the VM, e.g. the owner + * @param imageName Name of the VM + * @return File name for the VM derived from the function's input + */ + public static String vmName(UserInfo user, String imageName) { + return cleanFileName(vmNameDateFormat.print(System.currentTimeMillis()) + "_" + user.lastName + "_" + + imageName); + } + + /** + * Make sure file name contains only a subset of ascii characters and is not + * too long. + * + * @param name What we want to turn into a file name + * @return A sanitized form of name that should be safe to use as a file + * name + */ + public static String cleanFileName(String name) { + if (name == null) + return "null"; + name = name.replaceAll("[^a-zA-Z0-9_\\.\\-]+", "_"); + if (name.length() > 120) + name = name.substring(0, 120); + return name; + } + + public static String userFullName(UserInfo ui) { + return ui.firstName + " " + ui.lastName; + } +} diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/util/QuickTimer.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/util/QuickTimer.java new file mode 100644 index 00000000..7a317ff7 --- /dev/null +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/util/QuickTimer.java @@ -0,0 +1,38 @@ +package org.openslx.bwlp.sat.util; + +import java.util.Timer; +import java.util.TimerTask; + +/** + * This is a global, static {@link Timer} you can use anywhere for repeating + * tasks that will <b>not take a significant amount of time to execute</b>. This + * means they should not run any complex data base queries (better yet, none at + * all) or do heavy file I/O, etc.. + * The main reason for this class is to prevent having {@link Timer} threads + * everywhere in the server for trivial tasks. + */ +public class QuickTimer { + + private static final Timer timer = new Timer("QuickTimer"); + + public static void scheduleAtFixedDelay(TimerTask task, long delay, long period) { + timer.schedule(task, delay, period); + } + + public static void scheduleAtFixedRate(TimerTask task, long delay, long period) { + timer.scheduleAtFixedRate(task, delay, period); + } + + public static void scheduleOnce(TimerTask task, long delay) { + timer.schedule(task, delay); + } + + /** + * Cancel this timer. Should only be called when the server is shutting + * down. + */ + public static void cancel() { + timer.cancel(); + } + +} diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/util/Util.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/util/Util.java new file mode 100644 index 00000000..338ed325 --- /dev/null +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/util/Util.java @@ -0,0 +1,45 @@ +package org.openslx.bwlp.sat.util; + +import java.io.Closeable; + +public class Util { + + /** + * Try to close everything passed to this method. Never throw an exception + * if it fails, just silently continue. + * + * @param item One or more objects that are Closeable + */ + public static void safeClose(Closeable... item) { + if (item == null) + return; + for (Closeable c : item) { + if (c == null) + continue; + try { + c.close(); + } catch (Exception e) { + } + } + } + + /** + * Try to close everything passed to this method. Never throw an exception + * if it fails, just silently continue. + * + * @param item One or more objects that are AutoCloseable + */ + public static void safeClose(AutoCloseable... item) { + if (item == null) + return; + for (AutoCloseable c : item) { + if (c == null) + continue; + try { + c.close(); + } catch (Exception e) { + } + } + } + +} |