From e0005ceecfd9281230c4add7575b18ee88307774 Mon Sep 17 00:00:00 2001 From: Simon Rettberg Date: Thu, 11 Jun 2015 18:40:49 +0200 Subject: [server] On mah way (lots of restructuring, some early db classes, sql dump of current schema) --- .../src/main/java/fileserv/ActiveUpload.java | 209 --------------- .../src/main/java/fileserv/ChunkList.java | 78 ------ .../src/main/java/fileserv/FileChunk.java | 66 ----- .../src/main/java/fileserv/FileServer.java | 107 -------- .../src/main/java/models/Configuration.java | 72 ----- .../src/main/java/org/openslx/bwlp/sat/App.java | 93 +++++++ .../org/openslx/bwlp/sat/database/Database.java | 120 +++++++++ .../openslx/bwlp/sat/database/MysqlConnection.java | 77 ++++++ .../openslx/bwlp/sat/database/MysqlStatement.java | 289 ++++++++++++++++++++ .../openslx/bwlp/sat/database/mappers/DbImage.java | 62 +++++ .../sat/database/mappers/DbImagePermissions.java | 64 +++++ .../bwlp/sat/database/mappers/DbOrganization.java | 34 +++ .../openslx/bwlp/sat/fileserv/ActiveUpload.java | 207 +++++++++++++++ .../org/openslx/bwlp/sat/fileserv/ChunkList.java | 78 ++++++ .../org/openslx/bwlp/sat/fileserv/FileChunk.java | 66 +++++ .../org/openslx/bwlp/sat/fileserv/FileServer.java | 106 ++++++++ .../openslx/bwlp/sat/thrift/BinaryListener.java | 65 +++++ .../org/openslx/bwlp/sat/thrift/ServerHandler.java | 214 +++++++++++++++ .../openslx/bwlp/sat/thrift/SessionManager.java | 101 +++++++ .../openslx/bwlp/sat/thrift/cache/CachedList.java | 33 +++ .../bwlp/sat/thrift/cache/OperatingSystemList.java | 26 ++ .../bwlp/sat/thrift/cache/OrganizationList.java | 29 ++ .../org/openslx/bwlp/sat/util/Configuration.java | 72 +++++ .../java/org/openslx/bwlp/sat/util/Constants.java | 21 ++ .../java/org/openslx/bwlp/sat/util/FileSystem.java | 25 ++ .../java/org/openslx/bwlp/sat/util/Formatter.java | 57 ++++ .../java/org/openslx/bwlp/sat/util/QuickTimer.java | 38 +++ .../main/java/org/openslx/bwlp/sat/util/Util.java | 45 ++++ .../src/main/java/server/BinaryListener.java | 49 ---- .../src/main/java/server/ServerHandler.java | 215 --------------- .../src/main/java/server/SessionManager.java | 101 ------- .../src/main/java/server/StartServer.java | 62 ----- .../src/main/java/server/TBinaryProtocolSafe.java | 123 --------- .../src/main/java/sql/MysqlConnection.java | 74 ------ .../src/main/java/sql/MysqlStatement.java | 291 --------------------- dozentenmodulserver/src/main/java/sql/SQL.java | 81 ------ .../src/main/java/sql/models/DbImage.java | 41 --- .../src/main/java/thrift/OperatingSystemList.java | 37 --- .../src/main/java/util/Constants.java | 21 -- .../src/main/java/util/FileSystem.java | 25 -- .../src/main/java/util/Formatter.java | 59 ----- dozentenmodulserver/src/main/java/util/Util.java | 45 ---- 42 files changed, 1922 insertions(+), 1756 deletions(-) delete mode 100644 dozentenmodulserver/src/main/java/fileserv/ActiveUpload.java delete mode 100644 dozentenmodulserver/src/main/java/fileserv/ChunkList.java delete mode 100644 dozentenmodulserver/src/main/java/fileserv/FileChunk.java delete mode 100644 dozentenmodulserver/src/main/java/fileserv/FileServer.java delete mode 100644 dozentenmodulserver/src/main/java/models/Configuration.java create mode 100644 dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/App.java create mode 100644 dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/Database.java create mode 100644 dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/MysqlConnection.java create mode 100644 dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/MysqlStatement.java create mode 100644 dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbImage.java create mode 100644 dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbImagePermissions.java create mode 100644 dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbOrganization.java create mode 100644 dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/fileserv/ActiveUpload.java create mode 100644 dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/fileserv/ChunkList.java create mode 100644 dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/fileserv/FileChunk.java create mode 100644 dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/fileserv/FileServer.java create mode 100644 dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/thrift/BinaryListener.java create mode 100644 dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/thrift/ServerHandler.java create mode 100644 dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/thrift/SessionManager.java create mode 100644 dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/thrift/cache/CachedList.java create mode 100644 dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/thrift/cache/OperatingSystemList.java create mode 100644 dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/thrift/cache/OrganizationList.java create mode 100644 dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/util/Configuration.java create mode 100644 dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/util/Constants.java create mode 100644 dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/util/FileSystem.java create mode 100644 dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/util/Formatter.java create mode 100644 dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/util/QuickTimer.java create mode 100644 dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/util/Util.java delete mode 100644 dozentenmodulserver/src/main/java/server/BinaryListener.java delete mode 100644 dozentenmodulserver/src/main/java/server/ServerHandler.java delete mode 100644 dozentenmodulserver/src/main/java/server/SessionManager.java delete mode 100644 dozentenmodulserver/src/main/java/server/StartServer.java delete mode 100644 dozentenmodulserver/src/main/java/server/TBinaryProtocolSafe.java delete mode 100644 dozentenmodulserver/src/main/java/sql/MysqlConnection.java delete mode 100644 dozentenmodulserver/src/main/java/sql/MysqlStatement.java delete mode 100644 dozentenmodulserver/src/main/java/sql/SQL.java delete mode 100644 dozentenmodulserver/src/main/java/sql/models/DbImage.java delete mode 100644 dozentenmodulserver/src/main/java/thrift/OperatingSystemList.java delete mode 100644 dozentenmodulserver/src/main/java/util/Constants.java delete mode 100644 dozentenmodulserver/src/main/java/util/FileSystem.java delete mode 100644 dozentenmodulserver/src/main/java/util/Formatter.java delete mode 100644 dozentenmodulserver/src/main/java/util/Util.java (limited to 'dozentenmodulserver/src/main/java') diff --git a/dozentenmodulserver/src/main/java/fileserv/ActiveUpload.java b/dozentenmodulserver/src/main/java/fileserv/ActiveUpload.java deleted file mode 100644 index 334345f3..00000000 --- a/dozentenmodulserver/src/main/java/fileserv/ActiveUpload.java +++ /dev/null @@ -1,209 +0,0 @@ -package fileserv; - -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.RandomAccessFile; -import java.nio.ByteBuffer; -import java.util.List; -import java.util.UUID; -import java.util.concurrent.ThreadPoolExecutor; - -import models.Configuration; - -import org.apache.log4j.Logger; -import org.openslx.bwlp.thrift.iface.ImageDetailsRead; -import org.openslx.bwlp.thrift.iface.UserInfo; -import org.openslx.filetransfer.DataReceivedCallback; -import org.openslx.filetransfer.Downloader; -import org.openslx.filetransfer.FileRange; -import org.openslx.filetransfer.WantRangeCallback; - -import util.FileSystem; -import util.Formatter; - -public class ActiveUpload { - private static final Logger LOGGER = Logger.getLogger(ActiveUpload.class); - - /** - * This is an active upload, so on our end, we have a Downloader. - */ - private Downloader download = null; - - private final File destinationFile; - - private final RandomAccessFile outFile; - - private final ChunkList chunks; - - private final long fileSize; - - /** - * User owning this uploaded file. - */ - private final UserInfo owner; - - /** - * Base image this upload is a new version for. - */ - private final ImageDetailsRead image; - - // TODO: Use HashList for verification - - public ActiveUpload(UserInfo owner, ImageDetailsRead image, File destinationFile, long fileSize, - List sha1Sums) throws FileNotFoundException { - this.destinationFile = destinationFile; - this.outFile = new RandomAccessFile(destinationFile, "rw"); - this.chunks = new ChunkList(fileSize, sha1Sums); - this.owner = owner; - this.image = image; - this.fileSize = fileSize; - } - - /** - * Add another connection for this file transfer. Currently only one - * connection is allowed, but this might change in the future. - * - * @param connection - * @return true if the connection is accepted, false if it should be - * discarded - */ - public synchronized boolean addConnection(Downloader connection, ThreadPoolExecutor pool) { - if (download != null || chunks.isComplete()) - return false; - download = connection; - pool.execute(new Runnable() { - @Override - public void run() { - CbHandler cbh = new CbHandler(); - if (!download.download(cbh, cbh) && cbh.currentChunk != null) { - // If the download failed and we have a current chunk, put it back into - // the queue, so it will be handled again later... - chunks.markFailed(cbh.currentChunk); - } - } - }); - return true; - } - - /** - * Write some data to the local file. Thread safe so we could - * have multiple concurrent connections later. - * - * @param fileOffset - * @param dataLength - * @param data - * @return - */ - private boolean writeFileData(long fileOffset, int dataLength, byte[] data) { - synchronized (outFile) { - try { - outFile.seek(fileOffset); - outFile.write(data, 0, dataLength); - } catch (IOException e) { - LOGGER.error("Cannot write to '" + destinationFile - + "'. Disk full, network storage error, bad permissions, ...?", e); - return false; - } - } - return true; - } - - private void finishUpload() { - File file = destinationFile; - // Ready to go. First step: Rename temp file to something usable - File destination = new File(file.getParent(), Formatter.vmName(owner, image.imageName)); - // Sanity check: destination should be a sub directory of the vmStorePath - String relPath = FileSystem.getRelativePath(destination, Configuration.getVmStoreBasePath()); - if (relPath == null) { - LOGGER.warn(destination.getAbsolutePath() + " is not a subdir of " - + Configuration.getVmStoreBasePath().getAbsolutePath()); - // TODO: Update state to failed... - } - - // Execute rename - boolean ret = false; - Exception renameException = null; - try { - ret = file.renameTo(destination); - } catch (Exception e) { - ret = false; - renameException = e; - } - if (!ret) { - // Rename failed :-( - LOGGER.warn( - "Could not rename '" + file.getAbsolutePath() + "' to '" + destination.getAbsolutePath() - + "'", renameException); - // TODO: Update state.... - } - - // Now insert meta data into DB - - final String imageVersionId = UUID.randomUUID().toString(); - - // TODO: SQL magic, update state - } - - public void cancel() { - // TODO Auto-generated method stub - - } - - /** - * Get user owning this upload. Can be null in special cases. - * - * @return instance of UserInfo for the according user. - */ - public UserInfo getOwner() { - return this.owner; - } - - public boolean isComplete() { - return chunks.isComplete() && destinationFile.length() == this.fileSize; - } - - public File getDestinationFile() { - return this.destinationFile; - } - - public long getSize() { - return this.fileSize; - } - - /** - * Callback class for an instance of the Downloader, which supplies - * the Downloader with wanted file ranges, and handles incoming data. - */ - private class CbHandler implements WantRangeCallback, DataReceivedCallback { - /** - * The current chunk being transfered. - */ - public FileChunk currentChunk = null; - - @Override - public boolean dataReceived(long fileOffset, int dataLength, byte[] data) { - // TODO: Maybe cache in RAM and write full CHUNK_SIZE blocks at a time? - // Would probably help with slower storage, especially if it's using - // rotating disks and we're running multiple uploads. - // Also we wouldn't have to re-read a block form disk for sha1 checking. - return writeFileData(fileOffset, dataLength, data); - } - - @Override - public FileRange get() { - if (currentChunk != null) { - // TODO: A chunk was requested before, check hash and requeue if not matching - // This needs to be async (own thread) so will be a little complicated - } - // Get next missing chunk - currentChunk = chunks.getMissing(); - if (currentChunk == null) - return null; // No more chunks, returning null tells the Downloader we're done. - return currentChunk.range; - } - } - - // TODO: Clean up old stale uploads - -} diff --git a/dozentenmodulserver/src/main/java/fileserv/ChunkList.java b/dozentenmodulserver/src/main/java/fileserv/ChunkList.java deleted file mode 100644 index 95b3e1fa..00000000 --- a/dozentenmodulserver/src/main/java/fileserv/ChunkList.java +++ /dev/null @@ -1,78 +0,0 @@ -package fileserv; - -import java.nio.ByteBuffer; -import java.util.LinkedList; -import java.util.List; - -import org.apache.log4j.Logger; - -public class ChunkList { - - private static final Logger LOGGER = Logger.getLogger(ChunkList.class); - - /** - * Chunks that are missing from the file - */ - private final List missingChunks = new LinkedList<>(); - - /** - * Chunks that are currently being uploaded or hash-checked - */ - private final List pendingChunks = new LinkedList<>(); - - // Do we need to keep valid chunks, or chunks that failed too many times? - - public ChunkList(long fileSize, List sha1Sums) { - FileChunk.createChunkList(missingChunks, fileSize, sha1Sums); - } - - /** - * Get a missing chunk, marking it pending. - * - * @return chunk marked as missing - */ - public synchronized FileChunk getMissing() { - if (missingChunks.isEmpty()) - return null; - FileChunk c = missingChunks.remove(0); - pendingChunks.add(c); - return c; - } - - /** - * Mark a chunk currently transferring as successfully transfered. - * - * @param c The chunk in question - */ - public synchronized void markSuccessful(FileChunk c) { - if (!pendingChunks.remove(c)) { - LOGGER.warn("Inconsistent state: markTransferred called for Chunk " + c.toString() - + ", but chunk is not marked as currently transferring!"); - return; - } - } - - /** - * Mark a chunk currently transferring or being hash checked as failed - * transfer. This increases its fail count and re-adds it to the list of - * missing chunks. - * - * @param c The chunk in question - * @return Number of times transfer of this chunk failed - */ - public synchronized int markFailed(FileChunk c) { - if (!pendingChunks.remove(c)) { - LOGGER.warn("Inconsistent state: markTransferred called for Chunk " + c.toString() - + ", but chunk is not marked as currently transferring!"); - return -1; - } - // Add as first element so it will be re-transmitted immediately - missingChunks.add(0, c); - return c.incFailed(); - } - - public synchronized boolean isComplete() { - return missingChunks.isEmpty() && pendingChunks.isEmpty(); - } - -} diff --git a/dozentenmodulserver/src/main/java/fileserv/FileChunk.java b/dozentenmodulserver/src/main/java/fileserv/FileChunk.java deleted file mode 100644 index 1a95d27c..00000000 --- a/dozentenmodulserver/src/main/java/fileserv/FileChunk.java +++ /dev/null @@ -1,66 +0,0 @@ -package fileserv; - -import java.nio.ByteBuffer; -import java.util.Collection; -import java.util.List; - -import org.openslx.filetransfer.FileRange; - -public class FileChunk { - - public static final int CHUNK_SIZE_MIB = 16; - public static final int CHUNK_SIZE = CHUNK_SIZE_MIB * (1024 * 1024); - - public final FileRange range; - public final byte[] sha1sum; - private int failCount = 0; - - public FileChunk(long startOffset, long endOffset, byte[] sha1sum) { - this.range = new FileRange(startOffset, endOffset); - this.sha1sum = sha1sum; - } - - /** - * Signal that transferring this chunk seems to have failed (checksum - * mismatch). - * - * @return Number of times the transfer failed now - */ - public synchronized int incFailed() { - return ++failCount; - } - - // - - public static int fileSizeToChunkCount(long fileSize) { - return (int) ((fileSize + CHUNK_SIZE - 1) / CHUNK_SIZE); - } - - public static void createChunkList(Collection list, long fileSize, List sha1Sums) { - if (fileSize < 0) - throw new IllegalArgumentException("fileSize cannot be negative"); - long chunkCount = fileSizeToChunkCount(fileSize); - if (sha1Sums != null) { - if (sha1Sums.size() != chunkCount) - throw new IllegalArgumentException( - "Passed a sha1sum list, but hash count in list doesn't match expected chunk count"); - long offset = 0; - for (ByteBuffer sha1sum : sha1Sums) { // Do this as we don't know how efficient List.get(index) is... - long end = offset + CHUNK_SIZE; - if (end > fileSize) - end = fileSize; - list.add(new FileChunk(offset, end, sha1sum.array())); - offset = end; - } - return; - } - long offset = 0; - while (offset < fileSize) { // ...otherwise we could share this code - long end = offset + CHUNK_SIZE; - if (end > fileSize) - end = fileSize; - list.add(new FileChunk(offset, end, null)); - offset = end; - } - } -} diff --git a/dozentenmodulserver/src/main/java/fileserv/FileServer.java b/dozentenmodulserver/src/main/java/fileserv/FileServer.java deleted file mode 100644 index e82fa39c..00000000 --- a/dozentenmodulserver/src/main/java/fileserv/FileServer.java +++ /dev/null @@ -1,107 +0,0 @@ -package fileserv; - -import java.io.File; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.UUID; -import java.util.concurrent.ConcurrentHashMap; - -import org.openslx.bwlp.thrift.iface.TTransferRejectedException; -import org.openslx.bwlp.thrift.iface.UserInfo; -import org.openslx.filetransfer.Downloader; -import org.openslx.filetransfer.IncomingEvent; -import org.openslx.filetransfer.Listener; -import org.openslx.filetransfer.Uploader; - -import util.Constants; -import util.Formatter; - -public class FileServer implements IncomingEvent { - - /** - * Listener for incoming unencrypted connections - */ - private Listener plainListener = new Listener(this, null, 9092); // TODO: Config - - /** - * All currently running uploads, indexed by token - */ - private Map uploads = new ConcurrentHashMap<>(); - - private static final FileServer globalInstance = new FileServer(); - - private FileServer() { - } - - public static FileServer instance() { - return globalInstance; - } - - public boolean start() { - boolean ret = plainListener.start(); - // TODO: Start SSL listener too - return ret; - } - - @Override - public void incomingDownloadRequest(Uploader uploader) throws IOException { - // TODO Auto-generated method stub - - } - - @Override - public void incomingUploadRequest(Downloader downloader) throws IOException { - // TODO Auto-generated method stub - - } - - /** - * Get an upload instance by given token. - * - * @param uploadToken - * @return - */ - public ActiveUpload getUploadByToken(String uploadToken) { - return uploads.get(uploadToken); - } - - public String createNewUserUpload(UserInfo owner, long fileSize, List sha1Sums) - throws TTransferRejectedException, FileNotFoundException { - Iterator it = uploads.values().iterator(); - int activeUploads = 0; - while (it.hasNext()) { - ActiveUpload upload = it.next(); - if (upload.isComplete()) { - // TODO: Check age (short timeout) and remove - continue; - } else { - // Check age (long timeout) and remove - } - activeUploads++; - } - if (activeUploads > Constants.MAX_UPLOADS) - throw new TTransferRejectedException("Server busy. Too many running uploads."); - File destinationFile = null; - do { - destinationFile = Formatter.getTempImageName(); - } while (destinationFile.exists()); - // TODO: Pass image - ActiveUpload upload = new ActiveUpload(owner, null, destinationFile, fileSize, sha1Sums); - String key = UUID.randomUUID().toString(); - uploads.put(key, upload); - return key; - } - - public int getPlainPort() { - return plainListener.getPort(); - } - - public int getSslPort() { - return 0; // TODO - } - -} diff --git a/dozentenmodulserver/src/main/java/models/Configuration.java b/dozentenmodulserver/src/main/java/models/Configuration.java deleted file mode 100644 index 244e542b..00000000 --- a/dozentenmodulserver/src/main/java/models/Configuration.java +++ /dev/null @@ -1,72 +0,0 @@ -package models; - -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.Properties; - -import org.apache.log4j.Logger; -import org.joda.time.format.DateTimeFormat; -import org.joda.time.format.DateTimeFormatter; - -public class Configuration { - - private static final Logger LOGGER = Logger.getLogger(Configuration.class); - private static final DateTimeFormatter subdirDate = DateTimeFormat.forPattern("yy-MM"); - - private static File vmStoreBasePath; - private static File vmStoreProdPath; - private static String dbUri; - private static String dbUsername; - private static String dbPassword; - - public static boolean load() throws IOException { - // Load configuration from java properties file - Properties prop = new Properties(); - InputStream in = new FileInputStream("./config.properties"); - try { - prop.load(in); - } finally { - in.close(); - } - - vmStoreBasePath = new File(prop.getProperty("vmstore.path")); - vmStoreProdPath = new File(vmStoreBasePath, "prod"); - dbUri = prop.getProperty("db.uri"); - dbUsername = prop.getProperty("db.username"); - dbPassword = prop.getProperty("db.password"); - - // Currently all fields are mandatory but there might be optional settings in the future - return vmStoreBasePath != null && dbUri != null && dbUsername != null && dbPassword != null; - } - - // Static ("real") fields - - public static File getVmStoreBasePath() { - return vmStoreBasePath; - } - - public static String getDbUri() { - return dbUri; - } - - public static String getDbUsername() { - return dbUsername; - } - - public static String getDbPassword() { - return dbPassword; - } - - public static File getVmStoreProdPath() { - return vmStoreProdPath; - } - - // Dynamically Computed fields - - public static File getCurrentVmStorePath() { - return new File(vmStoreProdPath, subdirDate.print(System.currentTimeMillis())); - } - -} diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/App.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/App.java new file mode 100644 index 00000000..8aac1fcb --- /dev/null +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/App.java @@ -0,0 +1,93 @@ +package org.openslx.bwlp.sat; + +import java.security.NoSuchAlgorithmException; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; +import java.util.TimerTask; + +import org.apache.log4j.BasicConfigurator; +import org.apache.log4j.Logger; +import org.apache.thrift.transport.TTransportException; +import org.openslx.bwlp.sat.database.Database; +import org.openslx.bwlp.sat.database.mappers.DbImage; +import org.openslx.bwlp.sat.fileserv.FileServer; +import org.openslx.bwlp.sat.thrift.BinaryListener; +import org.openslx.bwlp.sat.thrift.cache.OperatingSystemList; +import org.openslx.bwlp.sat.thrift.cache.OrganizationList; +import org.openslx.bwlp.sat.util.Configuration; +import org.openslx.bwlp.sat.util.QuickTimer; +import org.openslx.bwlp.thrift.iface.ImageSummaryRead; +import org.openslx.bwlp.thrift.iface.UserInfo; + +public class App { + + private static Logger log = Logger.getLogger(App.class); + + private static List servers = new ArrayList<>(); + + public static boolean DEBUG = false; + + public static void main(String[] args) throws TTransportException, NoSuchAlgorithmException { + //get going and show basic information in log file + BasicConfigurator.configure(); + if (args.length != 0 && args[0].equals("debug")) { + DEBUG = true; + } + log.info("****************************************************************"); + log.info("******************* Starting Application ***********************"); + log.info("****************************************************************"); + + // get Configuration + try { + log.info("Loading configuration"); + Configuration.load(); + } catch (Exception e1) { + log.fatal("Could not load configuration", e1); + System.exit(1); + } + + // Load useful things from master server + OrganizationList.get(); + //OperatingSystemList.get(); + + // Start file transfer server + if (!FileServer.instance().start()) { + log.error("Could not start internal file server."); + return; + } + // Start Server + Thread t; + t = new Thread(new BinaryListener(9090, false)); + servers.add(t); + t.start(); + // DEBUG + if (DEBUG) { + Database.printCharsetInformation(); + List allVisible = DbImage.getAllVisible(new UserInfo("bla", "blu", null, null, + null), null); + log.info("Got " + allVisible.size()); + QuickTimer.scheduleAtFixedDelay(new TimerTask() { + @Override + public void run() { + Database.printDebug(); + } + }, 100, 5000); + } + // Wait for servers + for (Thread wait : servers) { + boolean success = false; + while (!success) { + try { + wait.join(); + success = true; + } catch (InterruptedException e) { + // Do nothing... + } + } + } + QuickTimer.cancel(); + log.info(new Date() + " - all Servers shut down, exiting...\n"); + } + +} diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/Database.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/Database.java new file mode 100644 index 00000000..cfc6530b --- /dev/null +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/Database.java @@ -0,0 +1,120 @@ +package org.openslx.bwlp.sat.database; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.Collections; +import java.util.Queue; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; + +import org.apache.log4j.Logger; +import org.openslx.bwlp.sat.util.Configuration; + +public class Database { + + private static final Logger LOGGER = Logger.getLogger(Database.class); + /** + * Pool of available connections. + */ + private static final Queue pool = new ConcurrentLinkedQueue<>(); + + /** + * Set of connections currently handed out. + */ + private static final Set busyConnections = Collections.newSetFromMap(new ConcurrentHashMap()); + + static { + try { + Class.forName("com.mysql.jdbc.Driver").newInstance(); + } catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) { + LOGGER.fatal("Cannot get mysql JDBC driver!", e); + System.exit(1); + } + } + + /** + * Get a connection to the database. If there is a valid connection in the + * pool, it will be returned. Otherwise, a new connection is created. If + * there are more than 20 busy connections, null is returned. + * + * @return connection to database, or null + */ + public static MysqlConnection getConnection() { + MysqlConnection con; + for (;;) { + con = pool.poll(); + if (con == null) + break; + if (!con.isValid()) { + con.release(); + continue; + } + if (!busyConnections.add(con)) + throw new RuntimeException("Tried to hand out a busy connection!"); + return con; + } + // No pooled connection + if (busyConnections.size() > 20) { + LOGGER.warn("Too many open MySQL connections. Possible connection leak!"); + return null; + } + try { + // Create fresh connection + Connection rawConnection = DriverManager.getConnection(Configuration.getDbUri(), + Configuration.getDbUsername(), Configuration.getDbPassword()); + // By convention in our program we don't want auto commit + rawConnection.setAutoCommit(false); + // Wrap into our proxy + con = new MysqlConnection(rawConnection); + // Keep track of busy mysql connection + if (!busyConnections.add(con)) + throw new RuntimeException("Tried to hand out a busy connection!"); + return con; + } catch (SQLException e) { + LOGGER.info("Failed to connect to local mysql server", e); + } + return null; + } + + /** + * Called by a {@link MysqlConnection} when its close()-method + * is called, so the connection will be added to the pool of available + * connections again. + * + * @param connection + */ + static void returnConnection(MysqlConnection connection) { + if (!busyConnections.remove(connection)) + throw new RuntimeException("Tried to return a mysql connection to the pool that was not taken!"); + pool.add(connection); + } + + public static void printCharsetInformation() { + LOGGER.info("MySQL charset related variables:"); + try (MysqlConnection connection = Database.getConnection()) { + MysqlStatement stmt = connection.prepareStatement("SHOW VARIABLES LIKE :what"); + stmt.setString("what", "char%"); + ResultSet rs = stmt.executeQuery(); + while (rs.next()) { + LOGGER.info(rs.getString("Variable_name") + ": " + rs.getString("Value")); + } + stmt.setString("what", "collat%"); + rs = stmt.executeQuery(); + while (rs.next()) { + LOGGER.info(rs.getString("Variable_name") + ": " + rs.getString("Value")); + } + } catch (SQLException e) { + LOGGER.error("Query failed in Database.printCharsetInformation()", e); + } + LOGGER.info("End of variables"); + } + + public static void printDebug() { + LOGGER.info("Available: " + pool.size()); + LOGGER.info("Busy: " + busyConnections.size()); + } + +}// end class diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/MysqlConnection.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/MysqlConnection.java new file mode 100644 index 00000000..24aaf1e8 --- /dev/null +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/MysqlConnection.java @@ -0,0 +1,77 @@ +package org.openslx.bwlp.sat.database; + +import java.sql.Connection; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.log4j.Logger; + +public class MysqlConnection implements AutoCloseable { + + private static final Logger LOGGER = Logger.getLogger(MysqlConnection.class); + + private static final int CONNECTION_TIMEOUT_MS = 5 * 60 * 1000; + + private final long deadline = System.currentTimeMillis() + CONNECTION_TIMEOUT_MS; + + private final Connection rawConnection; + + private boolean hasPendingQueries = false; + + private List openStatements = new ArrayList<>(); + + MysqlConnection(Connection rawConnection) { + this.rawConnection = rawConnection; + } + + public MysqlStatement prepareStatement(String sql) throws SQLException { + if (!sql.startsWith("SELECT")) + hasPendingQueries = true; + MysqlStatement statement = new MysqlStatement(rawConnection, sql); + openStatements.add(statement); + return statement; + } + + public void commit() throws SQLException { + rawConnection.commit(); + hasPendingQueries = false; + } + + public void rollback() throws SQLException { + rawConnection.rollback(); + hasPendingQueries = false; + } + + boolean isValid() { + return System.currentTimeMillis() < deadline; + } + + @Override + public void close() { + if (hasPendingQueries) { + LOGGER.warn("Mysql connection had uncommited queries on .close()"); + try { + rawConnection.rollback(); + } catch (SQLException e) { + LOGGER.warn("Rolling back uncommited queries failed!", e); + } + } + if (!openStatements.isEmpty()) { + for (MysqlStatement statement : openStatements) { + statement.close(); + } + openStatements.clear(); + } + Database.returnConnection(this); + } + + void release() { + try { + rawConnection.close(); + } catch (SQLException e) { + // Nothing meaningful to do + } + } + +} diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/MysqlStatement.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/MysqlStatement.java new file mode 100644 index 00000000..3d5f9065 --- /dev/null +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/MysqlStatement.java @@ -0,0 +1,289 @@ +package org.openslx.bwlp.sat.database; + +import java.io.Closeable; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +/** + * Class for creating {@link PreparedStatement}s with named parameters. Based on + * Named Parameters for PreparedStatement + */ +public class MysqlStatement implements Closeable { + + private static final QueryCache cache = new QueryCache(); + + private final PreparsedQuery query; + + private final PreparedStatement statement; + + private final List openResultSets = new ArrayList<>(); + + MysqlStatement(Connection con, String sql) throws SQLException { + PreparsedQuery query; + synchronized (cache) { + query = cache.get(sql); + } + if (query == null) { + query = parse(sql); + synchronized (cache) { + cache.put(sql, query); + } + } + this.query = query; + this.statement = con.prepareStatement(query.sql); + } + + /** + * Returns the indexes for a parameter. + * + * @param name parameter name + * @return parameter indexes + * @throws IllegalArgumentException if the parameter does not exist + */ + private List getIndexes(String name) { + List indexes = query.indexMap.get(name); + if (indexes == null) { + throw new IllegalArgumentException("Parameter not found: " + name); + } + return indexes; + } + + /** + * Sets a parameter. + * + * @param name parameter name + * @param value parameter value + * @throws SQLException if an error occurred + * @throws IllegalArgumentException if the parameter does not exist + * @see PreparedStatement#setObject(int, java.lang.Object) + */ + public void setObject(String name, Object value) throws SQLException { + List indexes = getIndexes(name); + for (Integer index : indexes) { + statement.setObject(index, value); + } + } + + /** + * Sets a parameter. + * + * @param name parameter name + * @param value parameter value + * @throws SQLException if an error occurred + * @throws IllegalArgumentException if the parameter does not exist + * @see PreparedStatement#setString(int, java.lang.String) + */ + public void setString(String name, String value) throws SQLException { + List indexes = getIndexes(name); + for (Integer index : indexes) { + statement.setString(index, value); + } + } + + /** + * Sets a parameter. + * + * @param name parameter name + * @param value parameter value + * @throws SQLException if an error occurred + * @throws IllegalArgumentException if the parameter does not exist + * @see PreparedStatement#setInt(int, int) + */ + public void setInt(String name, int value) throws SQLException { + List indexes = getIndexes(name); + for (Integer index : indexes) { + statement.setInt(index, value); + } + } + + /** + * Sets a parameter. + * + * @param name parameter name + * @param value parameter value + * @throws SQLException if an error occurred + * @throws IllegalArgumentException if the parameter does not exist + * @see PreparedStatement#setInt(int, int) + */ + public void setLong(String name, long value) throws SQLException { + List indexes = getIndexes(name); + for (Integer index : indexes) { + statement.setLong(index, value); + } + } + + /** + * Executes the statement. + * + * @return true if the first result is a {@link ResultSet} + * @throws SQLException if an error occurred + * @see PreparedStatement#execute() + */ + public boolean execute() throws SQLException { + return statement.execute(); + } + + /** + * Executes the statement, which must be a query. + * + * @return the query results + * @throws SQLException if an error occurred + * @see PreparedStatement#executeQuery() + */ + public ResultSet executeQuery() throws SQLException { + ResultSet rs = statement.executeQuery(); + openResultSets.add(rs); + return rs; + } + + /** + * Executes the statement, which must be an SQL INSERT, UPDATE or DELETE + * statement; or an SQL statement that returns nothing, such as a DDL + * statement. + * + * @return number of rows affected + * @throws SQLException if an error occurred + * @see PreparedStatement#executeUpdate() + */ + public int executeUpdate() throws SQLException { + return statement.executeUpdate(); + } + + /** + * Closes the statement. + * + * @see Statement#close() + */ + @Override + public void close() { + for (ResultSet rs : openResultSets) { + try { + rs.close(); + } catch (SQLException e) { + // + } + } + try { + statement.close(); + } catch (SQLException e) { + // Nothing to do + } + } + + /** + * Adds the current set of parameters as a batch entry. + * + * @throws SQLException if something went wrong + */ + public void addBatch() throws SQLException { + statement.addBatch(); + } + + /** + * Executes all of the batched statements. + * + * See {@link Statement#executeBatch()} for details. + * + * @return update counts for each statement + * @throws SQLException if something went wrong + */ + public int[] executeBatch() throws SQLException { + return statement.executeBatch(); + } + + // static methods + + private static PreparsedQuery parse(String query) { + int length = query.length(); + StringBuffer parsedQuery = new StringBuffer(length); + Map> paramMap = new HashMap<>(); + boolean inSingleQuote = false; + boolean inDoubleQuote = false; + boolean hasBackslash = false; + int index = 1; + + for (int i = 0; i < length; i++) { + char c = query.charAt(i); + if (hasBackslash) { + // Last char was a backslash, so we ignore the current char + hasBackslash = false; + } else if (c == '\\') { + // This is a backslash, next char will be escaped + hasBackslash = true; + } else if (inSingleQuote) { + // End of quoted string + if (c == '\'') { + inSingleQuote = false; + } + } else if (inDoubleQuote) { + // End of quoted string + if (c == '"') { + inDoubleQuote = false; + } + } else { + // Not in string, look for named params + if (c == '\'') { + inSingleQuote = true; + } else if (c == '"') { + inDoubleQuote = true; + } else if (c == ':' && i + 1 < length && Character.isJavaIdentifierStart(query.charAt(i + 1))) { + int j = i + 2; + while (j < length && Character.isJavaIdentifierPart(query.charAt(j))) { + j++; + } + String name = query.substring(i + 1, j); + c = '?'; // replace the parameter with a question mark + i += name.length(); // skip past the end of the parameter + + List indexList = paramMap.get(name); + if (indexList == null) { + indexList = new ArrayList<>(); + paramMap.put(name, indexList); + } + indexList.add(new Integer(index)); + + index++; + } + } + parsedQuery.append(c); + } + + return new PreparsedQuery(parsedQuery.toString(), paramMap); + } + + // private helper classes + + private static class PreparsedQuery { + private final Map> indexMap; + private final String sql; + + public PreparsedQuery(String sql, Map> indexMap) { + this.sql = sql; + this.indexMap = indexMap; + } + } + + private static class QueryCache extends LinkedHashMap { + private static final long serialVersionUID = 1L; + + public QueryCache() { + super(30, (float) 0.75, true); + } + + @Override + protected boolean removeEldestEntry(Map.Entry eldest) { + return size() > 40; + } + } + +} diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbImage.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbImage.java new file mode 100644 index 00000000..b772edb4 --- /dev/null +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbImage.java @@ -0,0 +1,62 @@ +package org.openslx.bwlp.sat.database.mappers; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.List; + +import org.apache.log4j.Logger; +import org.openslx.bwlp.sat.database.Database; +import org.openslx.bwlp.sat.database.MysqlConnection; +import org.openslx.bwlp.sat.database.MysqlStatement; +import org.openslx.bwlp.thrift.iface.ImagePermissions; +import org.openslx.bwlp.thrift.iface.ImageSummaryRead; +import org.openslx.bwlp.thrift.iface.ShareMode; +import org.openslx.bwlp.thrift.iface.UserInfo; + +public class DbImage { + + private static final Logger LOGGER = Logger.getLogger(DbImage.class); + + public static List getAllVisible(UserInfo user, List tagSearch) { + try (MysqlConnection connection = Database.getConnection()) { + MysqlStatement stmt = connection.prepareStatement("SELECT" + + " i.imagebaseid, i.currentversionid, i.latestversionid, i.displayname," + + " i.osid, i.virtid, i.createtime, i.updatetime, i.ownerid," + + " i.sharemode, i.istemplate, i.canlinkdefault, i.candownloaddefault," + + " i.caneditdefault, i.canadmindefault," + + " cur.expiretime, cur.filesize, cur.isenabled, cur.isrestricted, cur.isvalid," + + " lat.uploaderid, lat.isprocessed," + + " perm.canlink, perm.candownload, perm.canedit, perm.canadmin" + + " FROM imagebase i" + + " LEFT JOIN imageversion cur ON (cur.imageversionid = i.currentversionid)" + + " LEFT JOIN imageversion lat ON (lat.imageversionid = i.latestversionid)" + + " LEFT JOIN imagepermission perm ON (i.imagebaseid = perm.imagebaseid AND perm.userid = :userid)"); + stmt.setString("userid", user.userId); + ResultSet rs = stmt.executeQuery(); + List list = new ArrayList<>(); + while (rs.next()) { + ImagePermissions defaultPermissions = DbImagePermissions.fromResultSetDefault(rs); + ImageSummaryRead entry = new ImageSummaryRead(rs.getString("imagebaseid"), + rs.getString("currentversionid"), rs.getString("latestversionid"), + rs.getString("displayname"), rs.getInt("osid"), rs.getString("virtid"), + rs.getLong("createtime"), rs.getLong("updatetime"), rs.getLong("expiretime"), + rs.getString("ownerid"), rs.getString("uploaderid"), + toShareMode(rs.getString("sharemode")), rs.getLong("filesize"), + rs.getByte("isrestricted") != 0, rs.getByte("isvalid") != 0, + rs.getByte("isprocessed") != 0, rs.getByte("istemplate") != 0, defaultPermissions); + entry.userPermissions = DbImagePermissions.fromResultSetUser(rs); + list.add(entry); + } + return list; + } catch (SQLException e) { + LOGGER.error("Query failed in DbImage.getAllVisible()", e); + return null; + } + } + + private static ShareMode toShareMode(String string) { + return ShareMode.valueOf(string); + } + +} diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbImagePermissions.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbImagePermissions.java new file mode 100644 index 00000000..e254b085 --- /dev/null +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbImagePermissions.java @@ -0,0 +1,64 @@ +package org.openslx.bwlp.sat.database.mappers; + +import java.sql.ResultSet; +import java.sql.SQLException; + +import org.openslx.bwlp.thrift.iface.ImagePermissions; + +public class DbImagePermissions { + + /** + * Build an instance of {@link ImagePermissions} by reading the given + * columns from the given {@link ResultSet}. If there are no permissions + * given in the ResultSet, null is returned. + * + * @param rs the {@link ResultSet} to read from + * @param canLink Name of the column to read the "can link" permission from + * @param canDownload Name of the column to read the "can download" + * permission from + * @param canEdit Name of the column to read the "can edit" permission from + * @param canAdmin Name of the column to read the "can admin" permission + * from + * @return instance of {@link ImagePermissions}, or null + * @throws SQLException + */ + private static ImagePermissions fromResultSet(ResultSet rs, String canLink, String canDownload, + String canEdit, String canAdmin) throws SQLException { + byte link = rs.getByte(canLink); + if (rs.wasNull()) + return null; + return new ImagePermissions(link != 0, rs.getByte(canDownload) != 0, rs.getByte(canEdit) != 0, + rs.getByte(canAdmin) != 0); + } + + /** + * Build an instance of {@link ImagePermissions} by reading the + * columns canlink, candownload, + * canedit, canadmin from the given + * {@link ResultSet}. If there are no permissions + * given in the ResultSet, null is returned. + * + * @param rs the {@link ResultSet} to read from + * @return instance of {@link ImagePermissions}, or null + * @throws SQLException + */ + public static ImagePermissions fromResultSetUser(ResultSet rs) throws SQLException { + return fromResultSet(rs, "canlink", "candownload", "canedit", "canadmin"); + } + + /** + * Build an instance of {@link ImagePermissions} by reading the + * columns canlinkdefault, candownloaddefault, + * caneditdefault, canadmindefault from the given + * {@link ResultSet}. If there are no permissions + * given in the ResultSet, null is returned. + * + * @param rs the {@link ResultSet} to read from + * @return instance of {@link ImagePermissions}, or null + * @throws SQLException + */ + public static ImagePermissions fromResultSetDefault(ResultSet rs) throws SQLException { + return fromResultSet(rs, "canlinkdefault", "candownloaddefault", "caneditdefault", "canadmindefault"); + } + +} diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbOrganization.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbOrganization.java new file mode 100644 index 00000000..cc401af9 --- /dev/null +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbOrganization.java @@ -0,0 +1,34 @@ +package org.openslx.bwlp.sat.database.mappers; + +import java.sql.SQLException; +import java.util.List; + +import org.apache.log4j.Logger; +import org.openslx.bwlp.sat.database.Database; +import org.openslx.bwlp.sat.database.MysqlConnection; +import org.openslx.bwlp.sat.database.MysqlStatement; +import org.openslx.bwlp.thrift.iface.Organization; + +public class DbOrganization { + + private static final Logger LOGGER = Logger.getLogger(DbOrganization.class); + + public static boolean storeOrganizations(List organizations) { + try (MysqlConnection connection = Database.getConnection()) { + MysqlStatement stmt = connection.prepareStatement("INSERT INTO organization" + + " (organizationid, displayname, canlogin) VALUES (:id, :name, 0)" + + " ON DUPLICATE KEY UPDATE displayname = VALUES(displayname)"); + for (Organization organization : organizations) { + stmt.setString("id", organization.organizationId); + stmt.setString("name", organization.displayName); + stmt.executeUpdate(); + } + connection.commit(); + return true; + } catch (SQLException e) { + LOGGER.error("Query failed in DbOrganization.storeOrganization()", e); + return false; + } + } + +} diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/fileserv/ActiveUpload.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/fileserv/ActiveUpload.java new file mode 100644 index 00000000..a2474587 --- /dev/null +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/fileserv/ActiveUpload.java @@ -0,0 +1,207 @@ +package org.openslx.bwlp.sat.fileserv; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.ByteBuffer; +import java.util.List; +import java.util.UUID; +import java.util.concurrent.ThreadPoolExecutor; + +import org.apache.log4j.Logger; +import org.openslx.bwlp.sat.util.Configuration; +import org.openslx.bwlp.sat.util.FileSystem; +import org.openslx.bwlp.sat.util.Formatter; +import org.openslx.bwlp.thrift.iface.ImageDetailsRead; +import org.openslx.bwlp.thrift.iface.UserInfo; +import org.openslx.filetransfer.DataReceivedCallback; +import org.openslx.filetransfer.Downloader; +import org.openslx.filetransfer.FileRange; +import org.openslx.filetransfer.WantRangeCallback; + +public class ActiveUpload { + private static final Logger LOGGER = Logger.getLogger(ActiveUpload.class); + + /** + * This is an active upload, so on our end, we have a Downloader. + */ + private Downloader download = null; + + private final File destinationFile; + + private final RandomAccessFile outFile; + + private final ChunkList chunks; + + private final long fileSize; + + /** + * User owning this uploaded file. + */ + private final UserInfo owner; + + /** + * Base image this upload is a new version for. + */ + private final ImageDetailsRead image; + + // TODO: Use HashList for verification + + public ActiveUpload(UserInfo owner, ImageDetailsRead image, File destinationFile, long fileSize, + List sha1Sums) throws FileNotFoundException { + this.destinationFile = destinationFile; + this.outFile = new RandomAccessFile(destinationFile, "rw"); + this.chunks = new ChunkList(fileSize, sha1Sums); + this.owner = owner; + this.image = image; + this.fileSize = fileSize; + } + + /** + * Add another connection for this file transfer. Currently only one + * connection is allowed, but this might change in the future. + * + * @param connection + * @return true if the connection is accepted, false if it should be + * discarded + */ + public synchronized boolean addConnection(Downloader connection, ThreadPoolExecutor pool) { + if (download != null || chunks.isComplete()) + return false; + download = connection; + pool.execute(new Runnable() { + @Override + public void run() { + CbHandler cbh = new CbHandler(); + if (!download.download(cbh, cbh) && cbh.currentChunk != null) { + // If the download failed and we have a current chunk, put it back into + // the queue, so it will be handled again later... + chunks.markFailed(cbh.currentChunk); + } + } + }); + return true; + } + + /** + * Write some data to the local file. Thread safe so we could + * have multiple concurrent connections later. + * + * @param fileOffset + * @param dataLength + * @param data + * @return + */ + private boolean writeFileData(long fileOffset, int dataLength, byte[] data) { + synchronized (outFile) { + try { + outFile.seek(fileOffset); + outFile.write(data, 0, dataLength); + } catch (IOException e) { + LOGGER.error("Cannot write to '" + destinationFile + + "'. Disk full, network storage error, bad permissions, ...?", e); + return false; + } + } + return true; + } + + private void finishUpload() { + File file = destinationFile; + // Ready to go. First step: Rename temp file to something usable + File destination = new File(file.getParent(), Formatter.vmName(owner, image.imageName)); + // Sanity check: destination should be a sub directory of the vmStorePath + String relPath = FileSystem.getRelativePath(destination, Configuration.getVmStoreBasePath()); + if (relPath == null) { + LOGGER.warn(destination.getAbsolutePath() + " is not a subdir of " + + Configuration.getVmStoreBasePath().getAbsolutePath()); + // TODO: Update state to failed... + } + + // Execute rename + boolean ret = false; + Exception renameException = null; + try { + ret = file.renameTo(destination); + } catch (Exception e) { + ret = false; + renameException = e; + } + if (!ret) { + // Rename failed :-( + LOGGER.warn( + "Could not rename '" + file.getAbsolutePath() + "' to '" + destination.getAbsolutePath() + + "'", renameException); + // TODO: Update state.... + } + + // Now insert meta data into DB + + final String imageVersionId = UUID.randomUUID().toString(); + + // TODO: SQL magic, update state + } + + public void cancel() { + // TODO Auto-generated method stub + + } + + /** + * Get user owning this upload. Can be null in special cases. + * + * @return instance of UserInfo for the according user. + */ + public UserInfo getOwner() { + return this.owner; + } + + public boolean isComplete() { + return chunks.isComplete() && destinationFile.length() == this.fileSize; + } + + public File getDestinationFile() { + return this.destinationFile; + } + + public long getSize() { + return this.fileSize; + } + + /** + * Callback class for an instance of the Downloader, which supplies + * the Downloader with wanted file ranges, and handles incoming data. + */ + private class CbHandler implements WantRangeCallback, DataReceivedCallback { + /** + * The current chunk being transfered. + */ + public FileChunk currentChunk = null; + + @Override + public boolean dataReceived(long fileOffset, int dataLength, byte[] data) { + // TODO: Maybe cache in RAM and write full CHUNK_SIZE blocks at a time? + // Would probably help with slower storage, especially if it's using + // rotating disks and we're running multiple uploads. + // Also we wouldn't have to re-read a block form disk for sha1 checking. + return writeFileData(fileOffset, dataLength, data); + } + + @Override + public FileRange get() { + if (currentChunk != null) { + // TODO: A chunk was requested before, check hash and requeue if not matching + // This needs to be async (own thread) so will be a little complicated + } + // Get next missing chunk + currentChunk = chunks.getMissing(); + if (currentChunk == null) + return null; // No more chunks, returning null tells the Downloader we're done. + return currentChunk.range; + } + } + + // TODO: Clean up old stale uploads + +} diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/fileserv/ChunkList.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/fileserv/ChunkList.java new file mode 100644 index 00000000..b07193c5 --- /dev/null +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/fileserv/ChunkList.java @@ -0,0 +1,78 @@ +package org.openslx.bwlp.sat.fileserv; + +import java.nio.ByteBuffer; +import java.util.LinkedList; +import java.util.List; + +import org.apache.log4j.Logger; + +public class ChunkList { + + private static final Logger LOGGER = Logger.getLogger(ChunkList.class); + + /** + * Chunks that are missing from the file + */ + private final List missingChunks = new LinkedList<>(); + + /** + * Chunks that are currently being uploaded or hash-checked + */ + private final List pendingChunks = new LinkedList<>(); + + // Do we need to keep valid chunks, or chunks that failed too many times? + + public ChunkList(long fileSize, List sha1Sums) { + FileChunk.createChunkList(missingChunks, fileSize, sha1Sums); + } + + /** + * Get a missing chunk, marking it pending. + * + * @return chunk marked as missing + */ + public synchronized FileChunk getMissing() { + if (missingChunks.isEmpty()) + return null; + FileChunk c = missingChunks.remove(0); + pendingChunks.add(c); + return c; + } + + /** + * Mark a chunk currently transferring as successfully transfered. + * + * @param c The chunk in question + */ + public synchronized void markSuccessful(FileChunk c) { + if (!pendingChunks.remove(c)) { + LOGGER.warn("Inconsistent state: markTransferred called for Chunk " + c.toString() + + ", but chunk is not marked as currently transferring!"); + return; + } + } + + /** + * Mark a chunk currently transferring or being hash checked as failed + * transfer. This increases its fail count and re-adds it to the list of + * missing chunks. + * + * @param c The chunk in question + * @return Number of times transfer of this chunk failed + */ + public synchronized int markFailed(FileChunk c) { + if (!pendingChunks.remove(c)) { + LOGGER.warn("Inconsistent state: markTransferred called for Chunk " + c.toString() + + ", but chunk is not marked as currently transferring!"); + return -1; + } + // Add as first element so it will be re-transmitted immediately + missingChunks.add(0, c); + return c.incFailed(); + } + + public synchronized boolean isComplete() { + return missingChunks.isEmpty() && pendingChunks.isEmpty(); + } + +} diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/fileserv/FileChunk.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/fileserv/FileChunk.java new file mode 100644 index 00000000..ffa033a5 --- /dev/null +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/fileserv/FileChunk.java @@ -0,0 +1,66 @@ +package org.openslx.bwlp.sat.fileserv; + +import java.nio.ByteBuffer; +import java.util.Collection; +import java.util.List; + +import org.openslx.filetransfer.FileRange; + +public class FileChunk { + + public static final int CHUNK_SIZE_MIB = 16; + public static final int CHUNK_SIZE = CHUNK_SIZE_MIB * (1024 * 1024); + + public final FileRange range; + public final byte[] sha1sum; + private int failCount = 0; + + public FileChunk(long startOffset, long endOffset, byte[] sha1sum) { + this.range = new FileRange(startOffset, endOffset); + this.sha1sum = sha1sum; + } + + /** + * Signal that transferring this chunk seems to have failed (checksum + * mismatch). + * + * @return Number of times the transfer failed now + */ + public synchronized int incFailed() { + return ++failCount; + } + + // + + public static int fileSizeToChunkCount(long fileSize) { + return (int) ((fileSize + CHUNK_SIZE - 1) / CHUNK_SIZE); + } + + public static void createChunkList(Collection list, long fileSize, List sha1Sums) { + if (fileSize < 0) + throw new IllegalArgumentException("fileSize cannot be negative"); + long chunkCount = fileSizeToChunkCount(fileSize); + if (sha1Sums != null) { + if (sha1Sums.size() != chunkCount) + throw new IllegalArgumentException( + "Passed a sha1sum list, but hash count in list doesn't match expected chunk count"); + long offset = 0; + for (ByteBuffer sha1sum : sha1Sums) { // Do this as we don't know how efficient List.get(index) is... + long end = offset + CHUNK_SIZE; + if (end > fileSize) + end = fileSize; + list.add(new FileChunk(offset, end, sha1sum.array())); + offset = end; + } + return; + } + long offset = 0; + while (offset < fileSize) { // ...otherwise we could share this code + long end = offset + CHUNK_SIZE; + if (end > fileSize) + end = fileSize; + list.add(new FileChunk(offset, end, null)); + offset = end; + } + } +} diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/fileserv/FileServer.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/fileserv/FileServer.java new file mode 100644 index 00000000..c357c292 --- /dev/null +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/fileserv/FileServer.java @@ -0,0 +1,106 @@ +package org.openslx.bwlp.sat.fileserv; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.ConcurrentHashMap; + +import org.openslx.bwlp.sat.util.Constants; +import org.openslx.bwlp.sat.util.Formatter; +import org.openslx.bwlp.thrift.iface.TTransferRejectedException; +import org.openslx.bwlp.thrift.iface.UserInfo; +import org.openslx.filetransfer.Downloader; +import org.openslx.filetransfer.IncomingEvent; +import org.openslx.filetransfer.Listener; +import org.openslx.filetransfer.Uploader; + +public class FileServer implements IncomingEvent { + + /** + * Listener for incoming unencrypted connections + */ + private Listener plainListener = new Listener(this, null, 9092); // TODO: Config + + /** + * All currently running uploads, indexed by token + */ + private Map uploads = new ConcurrentHashMap<>(); + + private static final FileServer globalInstance = new FileServer(); + + private FileServer() { + } + + public static FileServer instance() { + return globalInstance; + } + + public boolean start() { + boolean ret = plainListener.start(); + // TODO: Start SSL listener too + return ret; + } + + @Override + public void incomingDownloadRequest(Uploader uploader) throws IOException { + // TODO Auto-generated method stub + + } + + @Override + public void incomingUploadRequest(Downloader downloader) throws IOException { + // TODO Auto-generated method stub + + } + + /** + * Get an upload instance by given token. + * + * @param uploadToken + * @return + */ + public ActiveUpload getUploadByToken(String uploadToken) { + return uploads.get(uploadToken); + } + + public String createNewUserUpload(UserInfo owner, long fileSize, List sha1Sums) + throws TTransferRejectedException, FileNotFoundException { + Iterator it = uploads.values().iterator(); + int activeUploads = 0; + while (it.hasNext()) { + ActiveUpload upload = it.next(); + if (upload.isComplete()) { + // TODO: Check age (short timeout) and remove + continue; + } else { + // Check age (long timeout) and remove + } + activeUploads++; + } + if (activeUploads > Constants.MAX_UPLOADS) + throw new TTransferRejectedException("Server busy. Too many running uploads."); + File destinationFile = null; + do { + destinationFile = Formatter.getTempImageName(); + } while (destinationFile.exists()); + // TODO: Pass image + ActiveUpload upload = new ActiveUpload(owner, null, destinationFile, fileSize, sha1Sums); + String key = UUID.randomUUID().toString(); + uploads.put(key, upload); + return key; + } + + public int getPlainPort() { + return plainListener.getPort(); + } + + public int getSslPort() { + return 0; // TODO + } + +} diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/thrift/BinaryListener.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/thrift/BinaryListener.java new file mode 100644 index 00000000..70c47edb --- /dev/null +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/thrift/BinaryListener.java @@ -0,0 +1,65 @@ +package org.openslx.bwlp.sat.thrift; + +import java.security.NoSuchAlgorithmException; + +import org.apache.log4j.Logger; +import org.apache.thrift.protocol.TProtocolFactory; +import org.apache.thrift.server.THsHaServer; +import org.apache.thrift.server.TServer; +import org.apache.thrift.transport.TNonblockingServerSocket; +import org.apache.thrift.transport.TNonblockingServerTransport; +import org.apache.thrift.transport.TTransportException; +import org.openslx.bwlp.thrift.iface.SatelliteServer; +import org.openslx.thrifthelper.TBinaryProtocolSafe; + +public class BinaryListener implements Runnable { + private static final Logger log = Logger.getLogger(BinaryListener.class); + + private static final int MAX_MSG_LEN = 30 * 1000 * 1000; + private static final int MINWORKERTHREADS = 2; + private static final int MAXWORKERTHREADS = 64; + + private final SatelliteServer.Processor processor = new SatelliteServer.Processor( + new ServerHandler()); + private final TProtocolFactory protFactory = new TBinaryProtocolSafe.Factory(true, true); + + private final TServer server; + + public BinaryListener(int port, boolean secure) throws TTransportException, NoSuchAlgorithmException { + if (secure) + server = initSecure(port); + else + server = initNormal(port); + } + + @Override + public void run() { + log.info("Starting Listener"); + server.serve(); + log.fatal("Listener stopped unexpectedly"); + // TODO: Restart listener; if it fails, quit server so it will be restarted by the OS + } + + private TServer initSecure(int port) throws NoSuchAlgorithmException, TTransportException { + // TODO + return null; + } + + private TServer initNormal(int port) throws TTransportException { + final TNonblockingServerTransport serverTransport; + try { + serverTransport = new TNonblockingServerSocket(port); + log.fatal("Listening on port " + port + " (plain handler)"); + } catch (TTransportException e) { + log.fatal("Could not listen on port " + port + " (plain handler)"); + throw e; + } + THsHaServer.Args args = new THsHaServer.Args(serverTransport); + args.protocolFactory(protFactory); + args.processor(processor); + args.workerThreads(8); + args.maxReadBufferBytes = MAX_MSG_LEN; + return new THsHaServer(args); + } + +} diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/thrift/ServerHandler.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/thrift/ServerHandler.java new file mode 100644 index 00000000..cf26b510 --- /dev/null +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/thrift/ServerHandler.java @@ -0,0 +1,214 @@ +package org.openslx.bwlp.sat.thrift; + +import java.nio.ByteBuffer; +import java.util.List; +import java.util.Map; + +import org.apache.log4j.Logger; +import org.apache.thrift.TException; +import org.openslx.bwlp.sat.database.mappers.DbImage; +import org.openslx.bwlp.sat.fileserv.ActiveUpload; +import org.openslx.bwlp.sat.fileserv.FileServer; +import org.openslx.bwlp.sat.thrift.cache.OperatingSystemList; +import org.openslx.bwlp.thrift.iface.ImageBaseWrite; +import org.openslx.bwlp.thrift.iface.ImageDetailsRead; +import org.openslx.bwlp.thrift.iface.ImagePermissions; +import org.openslx.bwlp.thrift.iface.ImageSummaryRead; +import org.openslx.bwlp.thrift.iface.ImageVersionWrite; +import org.openslx.bwlp.thrift.iface.LecturePermissions; +import org.openslx.bwlp.thrift.iface.LectureRead; +import org.openslx.bwlp.thrift.iface.LectureSummary; +import org.openslx.bwlp.thrift.iface.LectureWrite; +import org.openslx.bwlp.thrift.iface.OperatingSystem; +import org.openslx.bwlp.thrift.iface.Organization; +import org.openslx.bwlp.thrift.iface.SatelliteServer; +import org.openslx.bwlp.thrift.iface.TAuthorizationException; +import org.openslx.bwlp.thrift.iface.TInvalidTokenException; +import org.openslx.bwlp.thrift.iface.TNotFoundException; +import org.openslx.bwlp.thrift.iface.TTransferRejectedException; +import org.openslx.bwlp.thrift.iface.TransferInformation; +import org.openslx.bwlp.thrift.iface.UploadStatus; +import org.openslx.bwlp.thrift.iface.UserInfo; +import org.openslx.bwlp.thrift.iface.Virtualizer; +import org.openslx.sat.thrift.version.Version; + +public class ServerHandler implements SatelliteServer.Iface { + + private static final Logger log = Logger.getLogger(ServerHandler.class); + + private static final FileServer fileServer = FileServer.instance(); + + @Override + public long getVersion() throws TException { + return Version.VERSION; + } + + @Override + public TransferInformation requestImageVersionUpload(String userToken, String imageBaseId, long fileSize, + List blockHashes) throws TTransferRejectedException, TAuthorizationException, + TException { + // TODO Auto-generated method stub + return null; + } + + @Override + public void cancelUpload(String uploadToken) throws TException { + ActiveUpload upload = fileServer.getUploadByToken(uploadToken); + if (upload != null) + upload.cancel(); + + } + + @Override + public UploadStatus queryUploadStatus(String uploadToken) throws TInvalidTokenException, TException { + // TODO Auto-generated method stub + return null; + } + + @Override + public TransferInformation requestDownload(String userToken, String imageVersionId) + throws TAuthorizationException, TException { + // TODO Auto-generated method stub + return null; + } + + @Override + public void cancelDownload(String downloadToken) throws TException { + // TODO Auto-generated method stub + + } + + @Override + public boolean isAuthenticated(String userToken) throws TException { + return SessionManager.get(userToken) != null; + } + + @Override + public void invalidateSession(String userToken) throws TException { + SessionManager.remove(userToken); + } + + @Override + public List getOperatingSystems() throws TException { + return OperatingSystemList.get(); + } + + @Override + public List getVirtualizers() throws TException { + // TODO Auto-generated method stub + return null; + } + + @Override + public List getAllOrganizations() throws TException { + // TODO Auto-generated method stub + return null; + } + + @Override + public List getImageList(String userToken, List tagSearch) + throws TAuthorizationException, TException { + UserInfo user = SessionManager.getOrFail(userToken); + return DbImage.getAllVisible(user, tagSearch); + } + + @Override + public ImageDetailsRead getImageDetails(String userToken, String imageBaseId) + throws TAuthorizationException, TNotFoundException, TException { + // TODO Auto-generated method stub + return null; + } + + @Override + public boolean updateImageBase(String userToken, String imageBaseId, ImageBaseWrite image) + throws TAuthorizationException, TException { + // TODO Auto-generated method stub + return false; + } + + @Override + public boolean updateImageVersion(String userToken, String imageVersionId, ImageVersionWrite image) + throws TAuthorizationException, TException { + // TODO Auto-generated method stub + return false; + } + + @Override + public boolean deleteImageVersion(String userToken, String imageVersionId) + throws TAuthorizationException, TNotFoundException, TException { + // TODO Auto-generated method stub + return false; + } + + @Override + public boolean writeImagePermissions(String userToken, String imageId, + Map permissions) throws TAuthorizationException, TNotFoundException, + TException { + // TODO Auto-generated method stub + return false; + } + + @Override + public Map getImagePermissions(String userToken, String imageBaseId) + throws TAuthorizationException, TNotFoundException, TException { + // TODO Auto-generated method stub + return null; + } + + @Override + public String createLecture(String userToken, LectureWrite lecture) throws TAuthorizationException, + TException { + // TODO Auto-generated method stub + return null; + } + + @Override + public boolean updateLecture(String userToken, String lectureId, LectureWrite lecture) + throws TAuthorizationException, TNotFoundException, TException { + // TODO Auto-generated method stub + return false; + } + + @Override + public List getLectureList(String userToken) throws TAuthorizationException, TException { + // TODO Auto-generated method stub + return null; + } + + @Override + public LectureRead getLectureDetails(String userToken, String lectureId) throws TAuthorizationException, + TNotFoundException, TException { + // TODO Auto-generated method stub + return null; + } + + @Override + public List getLecturesByImageVersion(String userToken, String imageVersionId) + throws TAuthorizationException, TNotFoundException, TException { + // TODO Auto-generated method stub + return null; + } + + @Override + public boolean deleteLecture(String userToken, String lectureId) throws TAuthorizationException, + TNotFoundException, TException { + // TODO Auto-generated method stub + return false; + } + + @Override + public boolean writeLecturePermissions(String userToken, String lectureId, + Map permissions) throws TAuthorizationException, TNotFoundException, + TException { + // TODO Auto-generated method stub + return false; + } + + @Override + public Map getLecturePermissions(String userToken, String lectureId) + throws TAuthorizationException, TNotFoundException, TException { + // TODO Auto-generated method stub + return null; + } + +}// end class diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/thrift/SessionManager.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/thrift/SessionManager.java new file mode 100644 index 00000000..bf444a20 --- /dev/null +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/thrift/SessionManager.java @@ -0,0 +1,101 @@ +package org.openslx.bwlp.sat.thrift; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.TimeUnit; + +import org.apache.log4j.Logger; +import org.openslx.bwlp.thrift.iface.AuthorizationError; +import org.openslx.bwlp.thrift.iface.TAuthorizationException; +import org.openslx.bwlp.thrift.iface.UserInfo; +import org.openslx.thrifthelper.ThriftManager; + +/** + * Manages user sessions. Mainly used to map tokens to users. + * + */ +public class SessionManager { + + private static final Logger LOGGER = Logger.getLogger(SessionManager.class); + + private static class Entry { + private static final long SESSION_TIMEOUT = TimeUnit.DAYS.toMillis(1); + private final UserInfo user; + private long validUntil; + + private Entry(UserInfo user) { + this.user = user; + this.validUntil = System.currentTimeMillis() + SESSION_TIMEOUT; + } + + public void touch(long now) { + this.validUntil = now + SESSION_TIMEOUT; + } + } + + // saves the current tokens and the mapped userdata, returning from the server + private static Map tokenManager = new ConcurrentHashMap<>(); + + /** + * Get the user corresponding to the given token. + * + * @param token user's token + * @return UserInfo for the matching user + * @throws TAuthorizationException if the token is not known or the session + * expired + */ + public static UserInfo getOrFail(String token) throws TAuthorizationException { + UserInfo ui = get(token); + if (ui != null) + return ui; + throw new TAuthorizationException(AuthorizationError.NOT_AUTHENTICATED, + "Your session token is not known to the server"); + } + + /** + * Get the user corresponding to the given token. Returns null if the token + * is not known, or the session already timed out. + * + * @param token user's token + * @return UserInfo for the matching user + */ + public static UserInfo get(String token) { + Entry e = tokenManager.get(token); + if (e != null) { + // User session already cached + final long now = System.currentTimeMillis(); + if (e.validUntil < now) { + tokenManager.remove(token); + return getRemote(token); + } + e.touch(now); + return e.user; + } + return getRemote(token); + } + + /** + * Remove session matching the given token + * + * @param token + */ + public static void remove(String token) { + tokenManager.remove(token); + } + + private static UserInfo getRemote(String token) { + UserInfo ui = null; + try { + ui = ThriftManager.getMasterClient().getUserFromToken(token); + } catch (Exception e) { + LOGGER.warn("Could not reach master server to query for user token of a client!", e); + } + if (ui == null) + return null; + tokenManager.put(token, new Entry(ui)); + return ui; + } + + // TODO: Clean map of old entries periodically + +} diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/thrift/cache/CachedList.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/thrift/cache/CachedList.java new file mode 100644 index 00000000..4c986fd2 --- /dev/null +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/thrift/cache/CachedList.java @@ -0,0 +1,33 @@ +package org.openslx.bwlp.sat.thrift.cache; + +import java.util.List; + +import org.apache.log4j.Logger; +import org.apache.thrift.TException; +import org.openslx.util.TimeoutReference; + + +public abstract class CachedList { + + private static final Logger LOGGER = Logger.getLogger(CachedList.class); + + private final TimeoutReference> cachedList = new TimeoutReference<>(600000, null); + + protected abstract List getCallback() throws TException; + + protected synchronized List getInternal() { + List list = cachedList.get(); + if (list == null) { + try { + list = getCallback(); + } catch (TException e) { + LOGGER.warn("Could not retrieve " + getClass().getSimpleName() + " list from master server", + e); + return null; + } + cachedList.set(list); + } + return list; + } + +} diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/thrift/cache/OperatingSystemList.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/thrift/cache/OperatingSystemList.java new file mode 100644 index 00000000..020ae4ff --- /dev/null +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/thrift/cache/OperatingSystemList.java @@ -0,0 +1,26 @@ +package org.openslx.bwlp.sat.thrift.cache; + +import java.util.List; + +import org.apache.thrift.TException; +import org.openslx.bwlp.thrift.iface.OperatingSystem; +import org.openslx.thrifthelper.ThriftManager; + +/** + * Holds the list of all known organizations. The list is synchronized with + * the master server. + */ +public class OperatingSystemList extends CachedList { + + private static final OperatingSystemList instance = new OperatingSystemList(); + + public static List get() { + return instance.getInternal(); + } + + @Override + protected List getCallback() throws TException { + return ThriftManager.getMasterClient().getOperatingSystems(); + } + +} diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/thrift/cache/OrganizationList.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/thrift/cache/OrganizationList.java new file mode 100644 index 00000000..8db7e7e5 --- /dev/null +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/thrift/cache/OrganizationList.java @@ -0,0 +1,29 @@ +package org.openslx.bwlp.sat.thrift.cache; + +import java.util.List; + +import org.apache.thrift.TException; +import org.openslx.bwlp.sat.database.mappers.DbOrganization; +import org.openslx.bwlp.thrift.iface.Organization; +import org.openslx.thrifthelper.ThriftManager; + +/** + * Holds the list of all known organizations. The list is synchronized with + * the master server. + */ +public class OrganizationList extends CachedList { + + private static final OrganizationList instance = new OrganizationList(); + + public static List get() { + return instance.getInternal(); + } + + @Override + protected List getCallback() throws TException { + List organizations = ThriftManager.getMasterClient().getOrganizations(); + DbOrganization.storeOrganizations(organizations); + return organizations; + } + +} diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/util/Configuration.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/util/Configuration.java new file mode 100644 index 00000000..07cd3a8d --- /dev/null +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/util/Configuration.java @@ -0,0 +1,72 @@ +package org.openslx.bwlp.sat.util; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.io.InputStream; +import java.util.Properties; + +import org.apache.log4j.Logger; +import org.joda.time.format.DateTimeFormat; +import org.joda.time.format.DateTimeFormatter; + +public class Configuration { + + private static final Logger LOGGER = Logger.getLogger(Configuration.class); + private static final DateTimeFormatter subdirDate = DateTimeFormat.forPattern("yy-MM"); + + private static File vmStoreBasePath; + private static File vmStoreProdPath; + private static String dbUri; + private static String dbUsername; + private static String dbPassword; + + public static boolean load() throws IOException { + // Load configuration from java properties file + Properties prop = new Properties(); + InputStream in = new FileInputStream("./config.properties"); + try { + prop.load(in); + } finally { + in.close(); + } + + vmStoreBasePath = new File(prop.getProperty("vmstore.path")); + vmStoreProdPath = new File(vmStoreBasePath, "prod"); + dbUri = prop.getProperty("db.uri"); + dbUsername = prop.getProperty("db.username"); + dbPassword = prop.getProperty("db.password"); + + // Currently all fields are mandatory but there might be optional settings in the future + return vmStoreBasePath != null && dbUri != null && dbUsername != null && dbPassword != null; + } + + // Static ("real") fields + + public static File getVmStoreBasePath() { + return vmStoreBasePath; + } + + public static String getDbUri() { + return dbUri; + } + + public static String getDbUsername() { + return dbUsername; + } + + public static String getDbPassword() { + return dbPassword; + } + + public static File getVmStoreProdPath() { + return vmStoreProdPath; + } + + // Dynamically Computed fields + + public static File getCurrentVmStorePath() { + return new File(vmStoreProdPath, subdirDate.print(System.currentTimeMillis())); + } + +} diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/util/Constants.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/util/Constants.java new file mode 100644 index 00000000..6c2dc31b --- /dev/null +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/util/Constants.java @@ -0,0 +1,21 @@ +package org.openslx.bwlp.sat.util; + +import org.openslx.bwlp.sat.fileserv.FileChunk; + +public class Constants { + public static final String INCOMPLETE_UPLOAD_SUFFIX = ".part"; + public static final int MAX_UPLOADS; + + static { + long maxMem = Runtime.getRuntime().maxMemory(); + if (maxMem == Long.MAX_VALUE) { + // Apparently the JVM was started without a memory limit (no -Xmx cmdline), + // so we assume a default of 256MB + maxMem = 256l * 1024l * 1024l; + } + maxMem /= 1024l * 1024l; + // Now maxMem is the amount of memory in MiB + + MAX_UPLOADS = (int) Math.max(1, (maxMem - 64) / (FileChunk.CHUNK_SIZE_MIB + 1)); + } +} diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/util/FileSystem.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/util/FileSystem.java new file mode 100644 index 00000000..38841cd9 --- /dev/null +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/util/FileSystem.java @@ -0,0 +1,25 @@ +package org.openslx.bwlp.sat.util; + +import java.io.File; + +import org.apache.log4j.Logger; + +public class FileSystem { + + private static final Logger LOGGER = Logger.getLogger(FileSystem.class); + + public static String getRelativePath(File absolutePath, File parentDir) { + String file; + String dir; + try { + file = absolutePath.getCanonicalPath(); + dir = parentDir.getCanonicalPath() + File.separator; + } catch (Exception e) { + LOGGER.error("Could not get relative path for " + absolutePath.toString(), e); + return null; + } + if (!file.startsWith(dir)) + return null; + return file.substring(dir.length()); + } +} diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/util/Formatter.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/util/Formatter.java new file mode 100644 index 00000000..0839ad24 --- /dev/null +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/util/Formatter.java @@ -0,0 +1,57 @@ +package org.openslx.bwlp.sat.util; + +import java.io.File; +import java.util.UUID; + +import org.joda.time.format.DateTimeFormat; +import org.joda.time.format.DateTimeFormatter; +import org.openslx.bwlp.thrift.iface.UserInfo; + +public class Formatter { + + private static final DateTimeFormatter vmNameDateFormat = DateTimeFormat.forPattern("dd_HH-mm-ss"); + + /** + * Generate a unique file name used for a virtual machine + * image that is currently uploading. + * + * @return Absolute path name of file + */ + public static File getTempImageName() { + return new File(Configuration.getCurrentVmStorePath(), UUID.randomUUID().toString() + + Constants.INCOMPLETE_UPLOAD_SUFFIX); + } + + /** + * Generate a file name for the given VM based on owner and display name. + * + * @param user The user associated with the VM, e.g. the owner + * @param imageName Name of the VM + * @return File name for the VM derived from the function's input + */ + public static String vmName(UserInfo user, String imageName) { + return cleanFileName(vmNameDateFormat.print(System.currentTimeMillis()) + "_" + user.lastName + "_" + + imageName); + } + + /** + * Make sure file name contains only a subset of ascii characters and is not + * too long. + * + * @param name What we want to turn into a file name + * @return A sanitized form of name that should be safe to use as a file + * name + */ + public static String cleanFileName(String name) { + if (name == null) + return "null"; + name = name.replaceAll("[^a-zA-Z0-9_\\.\\-]+", "_"); + if (name.length() > 120) + name = name.substring(0, 120); + return name; + } + + public static String userFullName(UserInfo ui) { + return ui.firstName + " " + ui.lastName; + } +} diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/util/QuickTimer.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/util/QuickTimer.java new file mode 100644 index 00000000..7a317ff7 --- /dev/null +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/util/QuickTimer.java @@ -0,0 +1,38 @@ +package org.openslx.bwlp.sat.util; + +import java.util.Timer; +import java.util.TimerTask; + +/** + * This is a global, static {@link Timer} you can use anywhere for repeating + * tasks that will not take a significant amount of time to execute. This + * means they should not run any complex data base queries (better yet, none at + * all) or do heavy file I/O, etc.. + * The main reason for this class is to prevent having {@link Timer} threads + * everywhere in the server for trivial tasks. + */ +public class QuickTimer { + + private static final Timer timer = new Timer("QuickTimer"); + + public static void scheduleAtFixedDelay(TimerTask task, long delay, long period) { + timer.schedule(task, delay, period); + } + + public static void scheduleAtFixedRate(TimerTask task, long delay, long period) { + timer.scheduleAtFixedRate(task, delay, period); + } + + public static void scheduleOnce(TimerTask task, long delay) { + timer.schedule(task, delay); + } + + /** + * Cancel this timer. Should only be called when the server is shutting + * down. + */ + public static void cancel() { + timer.cancel(); + } + +} diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/util/Util.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/util/Util.java new file mode 100644 index 00000000..338ed325 --- /dev/null +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/util/Util.java @@ -0,0 +1,45 @@ +package org.openslx.bwlp.sat.util; + +import java.io.Closeable; + +public class Util { + + /** + * Try to close everything passed to this method. Never throw an exception + * if it fails, just silently continue. + * + * @param item One or more objects that are Closeable + */ + public static void safeClose(Closeable... item) { + if (item == null) + return; + for (Closeable c : item) { + if (c == null) + continue; + try { + c.close(); + } catch (Exception e) { + } + } + } + + /** + * Try to close everything passed to this method. Never throw an exception + * if it fails, just silently continue. + * + * @param item One or more objects that are AutoCloseable + */ + public static void safeClose(AutoCloseable... item) { + if (item == null) + return; + for (AutoCloseable c : item) { + if (c == null) + continue; + try { + c.close(); + } catch (Exception e) { + } + } + } + +} diff --git a/dozentenmodulserver/src/main/java/server/BinaryListener.java b/dozentenmodulserver/src/main/java/server/BinaryListener.java deleted file mode 100644 index 92954322..00000000 --- a/dozentenmodulserver/src/main/java/server/BinaryListener.java +++ /dev/null @@ -1,49 +0,0 @@ -package server; - -import java.util.Date; - -import org.apache.log4j.Logger; -import org.apache.thrift.protocol.TProtocolFactory; -import org.apache.thrift.server.TServer; -import org.apache.thrift.server.TThreadPoolServer; -import org.apache.thrift.server.TThreadPoolServer.Args; -import org.apache.thrift.transport.TServerSocket; -import org.apache.thrift.transport.TServerTransport; -import org.apache.thrift.transport.TTransportException; -import org.openslx.bwlp.thrift.iface.SatelliteServer; - -public class BinaryListener implements Runnable { - private static Logger log = Logger.getLogger(BinaryListener.class); - - private final int MINWORKERTHREADS = 20; // keine ahnung ob das passt... - private final int MAXWORKERTHREADS = 80; // ebenso - - @Override - public void run() { - final ServerHandler handler = new ServerHandler(); - final SatelliteServer.Processor processor = new SatelliteServer.Processor( - handler); - final TServerTransport transport; - final TProtocolFactory protFactory = new TBinaryProtocolSafe.Factory( - true, true); - try { - transport = new TServerSocket(9090); - log.info(new Date() + " - Listening on Port 9090"); - - } catch (TTransportException e) { - log.fatal(new Date() + " - Could not listen on port 9090"); - return; - } - TServer server = new TThreadPoolServer(new Args(transport) - .protocolFactory(protFactory).processor(processor) - .minWorkerThreads(MINWORKERTHREADS) - .maxWorkerThreads(MAXWORKERTHREADS)); - - log.info(new Date() + " - Started running BinaryListener"); - log.info(new Date() + " - MINWORKERTHREADS=" + MINWORKERTHREADS - + " and MAXWORKERTHREADS=" + MAXWORKERTHREADS + "\n"); - server.serve(); - - } - -} diff --git a/dozentenmodulserver/src/main/java/server/ServerHandler.java b/dozentenmodulserver/src/main/java/server/ServerHandler.java deleted file mode 100644 index dddec9be..00000000 --- a/dozentenmodulserver/src/main/java/server/ServerHandler.java +++ /dev/null @@ -1,215 +0,0 @@ -package server; - -import java.nio.ByteBuffer; -import java.util.List; -import java.util.Map; - -import org.apache.log4j.Logger; -import org.apache.thrift.TException; -import org.openslx.bwlp.thrift.iface.ImageBaseWrite; -import org.openslx.bwlp.thrift.iface.ImageDetailsRead; -import org.openslx.bwlp.thrift.iface.ImagePermissions; -import org.openslx.bwlp.thrift.iface.ImageSummaryRead; -import org.openslx.bwlp.thrift.iface.ImageVersionWrite; -import org.openslx.bwlp.thrift.iface.LecturePermissions; -import org.openslx.bwlp.thrift.iface.LectureRead; -import org.openslx.bwlp.thrift.iface.LectureSummary; -import org.openslx.bwlp.thrift.iface.LectureWrite; -import org.openslx.bwlp.thrift.iface.OperatingSystem; -import org.openslx.bwlp.thrift.iface.Organization; -import org.openslx.bwlp.thrift.iface.SatelliteServer; -import org.openslx.bwlp.thrift.iface.TAuthorizationException; -import org.openslx.bwlp.thrift.iface.TInvalidTokenException; -import org.openslx.bwlp.thrift.iface.TNotFoundException; -import org.openslx.bwlp.thrift.iface.TTransferRejectedException; -import org.openslx.bwlp.thrift.iface.TransferInformation; -import org.openslx.bwlp.thrift.iface.UploadStatus; -import org.openslx.bwlp.thrift.iface.UserInfo; -import org.openslx.bwlp.thrift.iface.Virtualizer; -import org.openslx.sat.thrift.version.Version; - -import sql.models.DbImage; -import thrift.OperatingSystemList; -import fileserv.ActiveUpload; -import fileserv.FileServer; - -public class ServerHandler implements SatelliteServer.Iface { - - private static final Logger log = Logger.getLogger(ServerHandler.class); - - private static final FileServer fileServer = FileServer.instance(); - - @Override - public long getVersion() throws TException { - return Version.VERSION; - } - - @Override - public TransferInformation requestImageVersionUpload(String userToken, String imageBaseId, long fileSize, - List blockHashes) throws TTransferRejectedException, TAuthorizationException, - TException { - // TODO Auto-generated method stub - return null; - } - - @Override - public void cancelUpload(String uploadToken) throws TException { - ActiveUpload upload = fileServer.getUploadByToken(uploadToken); - if (upload != null) - upload.cancel(); - - } - - @Override - public UploadStatus queryUploadStatus(String uploadToken) throws TInvalidTokenException, TException { - // TODO Auto-generated method stub - return null; - } - - @Override - public TransferInformation requestDownload(String userToken, String imageVersionId) - throws TAuthorizationException, TException { - // TODO Auto-generated method stub - return null; - } - - @Override - public void cancelDownload(String downloadToken) throws TException { - // TODO Auto-generated method stub - - } - - @Override - public boolean isAuthenticated(String userToken) throws TException { - return SessionManager.get(userToken) != null; - } - - @Override - public void invalidateSession(String userToken) throws TException { - SessionManager.remove(userToken); - } - - @Override - public List getOperatingSystems() throws TException { - return OperatingSystemList.get(); - } - - @Override - public List getVirtualizers() throws TException { - // TODO Auto-generated method stub - return null; - } - - @Override - public List getAllOrganizations() throws TException { - // TODO Auto-generated method stub - return null; - } - - @Override - public List getImageList(String userToken, List tagSearch) - throws TAuthorizationException, TException { - UserInfo user = SessionManager.getOrFail(userToken); - return DbImage.getAllVisible(user, tagSearch); - } - - @Override - public ImageDetailsRead getImageDetails(String userToken, String imageBaseId) - throws TAuthorizationException, TNotFoundException, TException { - // TODO Auto-generated method stub - return null; - } - - @Override - public boolean updateImageBase(String userToken, String imageBaseId, ImageBaseWrite image) - throws TAuthorizationException, TException { - // TODO Auto-generated method stub - return false; - } - - @Override - public boolean updateImageVersion(String userToken, String imageVersionId, ImageVersionWrite image) - throws TAuthorizationException, TException { - // TODO Auto-generated method stub - return false; - } - - @Override - public boolean deleteImageVersion(String userToken, String imageVersionId) - throws TAuthorizationException, TNotFoundException, TException { - // TODO Auto-generated method stub - return false; - } - - @Override - public boolean writeImagePermissions(String userToken, String imageId, - Map permissions) throws TAuthorizationException, TNotFoundException, - TException { - // TODO Auto-generated method stub - return false; - } - - @Override - public Map getImagePermissions(String userToken, String imageBaseId) - throws TAuthorizationException, TNotFoundException, TException { - // TODO Auto-generated method stub - return null; - } - - @Override - public String createLecture(String userToken, LectureWrite lecture) throws TAuthorizationException, - TException { - // TODO Auto-generated method stub - return null; - } - - @Override - public boolean updateLecture(String userToken, String lectureId, LectureWrite lecture) - throws TAuthorizationException, TNotFoundException, TException { - // TODO Auto-generated method stub - return false; - } - - @Override - public List getLectureList(String userToken) throws TAuthorizationException, TException { - // TODO Auto-generated method stub - return null; - } - - @Override - public LectureRead getLectureDetails(String userToken, String lectureId) throws TAuthorizationException, - TNotFoundException, TException { - // TODO Auto-generated method stub - return null; - } - - @Override - public List getLecturesByImageVersion(String userToken, String imageVersionId) - throws TAuthorizationException, TNotFoundException, TException { - // TODO Auto-generated method stub - return null; - } - - @Override - public boolean deleteLecture(String userToken, String lectureId) throws TAuthorizationException, - TNotFoundException, TException { - // TODO Auto-generated method stub - return false; - } - - @Override - public boolean writeLecturePermissions(String userToken, String lectureId, - Map permissions) throws TAuthorizationException, TNotFoundException, - TException { - // TODO Auto-generated method stub - return false; - } - - @Override - public Map getLecturePermissions(String userToken, String lectureId) - throws TAuthorizationException, TNotFoundException, TException { - // TODO Auto-generated method stub - return null; - } - -}// end class diff --git a/dozentenmodulserver/src/main/java/server/SessionManager.java b/dozentenmodulserver/src/main/java/server/SessionManager.java deleted file mode 100644 index 3f4d4257..00000000 --- a/dozentenmodulserver/src/main/java/server/SessionManager.java +++ /dev/null @@ -1,101 +0,0 @@ -package server; - -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.TimeUnit; - -import org.apache.log4j.Logger; -import org.openslx.bwlp.thrift.iface.AuthorizationError; -import org.openslx.bwlp.thrift.iface.TAuthorizationException; -import org.openslx.bwlp.thrift.iface.UserInfo; -import org.openslx.thrifthelper.ThriftManager; - -/** - * Manages user sessions. Mainly used to map tokens to users. - * - */ -public class SessionManager { - - private static final Logger LOGGER = Logger.getLogger(SessionManager.class); - - private static class Entry { - private static final long SESSION_TIMEOUT = TimeUnit.DAYS.toMillis(1); - private final UserInfo user; - private long validUntil; - - private Entry(UserInfo user) { - this.user = user; - this.validUntil = System.currentTimeMillis() + SESSION_TIMEOUT; - } - - public void touch(long now) { - this.validUntil = now + SESSION_TIMEOUT; - } - } - - // saves the current tokens and the mapped userdata, returning from the server - private static Map tokenManager = new ConcurrentHashMap<>(); - - /** - * Get the user corresponding to the given token. - * - * @param token user's token - * @return UserInfo for the matching user - * @throws TAuthorizationException if the token is not known or the session - * expired - */ - public static UserInfo getOrFail(String token) throws TAuthorizationException { - UserInfo ui = get(token); - if (ui != null) - return ui; - throw new TAuthorizationException(AuthorizationError.NOT_AUTHENTICATED, - "Your session token is not known to the server"); - } - - /** - * Get the user corresponding to the given token. Returns null if the token - * is not known, or the session already timed out. - * - * @param token user's token - * @return UserInfo for the matching user - */ - public static UserInfo get(String token) { - Entry e = tokenManager.get(token); - if (e != null) { - // User session already cached - final long now = System.currentTimeMillis(); - if (e.validUntil < now) { - tokenManager.remove(token); - return getRemote(token); - } - e.touch(now); - return e.user; - } - return getRemote(token); - } - - /** - * Remove session matching the given token - * - * @param token - */ - public static void remove(String token) { - tokenManager.remove(token); - } - - private static UserInfo getRemote(String token) { - UserInfo ui = null; - try { - ui = ThriftManager.getMasterClient().getUserFromToken(token); - } catch (Exception e) { - LOGGER.warn("Could not reach master server to query for user token of a client!", e); - } - if (ui == null) - return null; - tokenManager.put(token, new Entry(ui)); - return ui; - } - - // TODO: Clean map of old entries periodically - -} diff --git a/dozentenmodulserver/src/main/java/server/StartServer.java b/dozentenmodulserver/src/main/java/server/StartServer.java deleted file mode 100644 index a5631622..00000000 --- a/dozentenmodulserver/src/main/java/server/StartServer.java +++ /dev/null @@ -1,62 +0,0 @@ -package server; - -import java.util.ArrayList; -import java.util.Date; -import java.util.List; - -import models.Configuration; - -import org.apache.log4j.BasicConfigurator; -import org.apache.log4j.Logger; - -import fileserv.FileServer; - -public class StartServer { - - private static Logger log = Logger.getLogger(StartServer.class); - - private static List servers = new ArrayList<>(); - - public static void main(String[] args) { - - //get going and show basic information in logfile - BasicConfigurator.configure(); - log.info("****************************************************************"); - log.info("******************* starting Application ***********************"); - log.info("****************************************************************"); - - // get Configuration - try { - log.info("Loading configuration"); - Configuration.load(); - } catch (Exception e1) { - log.fatal("Could not load configuration", e1); - System.exit(1); - } - - // Start file transfer server - if (!FileServer.instance().start()) { - log.error("Could not start internal file server."); - return; - } - // Start Server - Thread t; - t = new Thread(new BinaryListener()); - servers.add(t); - t.start(); - // Wait for servers - for (Thread wait : servers) { - boolean success = false; - while (!success) { - try { - wait.join(); - success = true; - } catch (InterruptedException e) { - // Do nothing... - } - } - } - log.info(new Date() + " - all Servers shut down, exiting...\n"); - } - -} diff --git a/dozentenmodulserver/src/main/java/server/TBinaryProtocolSafe.java b/dozentenmodulserver/src/main/java/server/TBinaryProtocolSafe.java deleted file mode 100644 index 843b58b1..00000000 --- a/dozentenmodulserver/src/main/java/server/TBinaryProtocolSafe.java +++ /dev/null @@ -1,123 +0,0 @@ -package server; - -import java.io.UnsupportedEncodingException; -import java.nio.ByteBuffer; - -import org.apache.thrift.TException; -import org.apache.thrift.protocol.TBinaryProtocol; -import org.apache.thrift.protocol.TMessage; -import org.apache.thrift.protocol.TProtocol; -import org.apache.thrift.protocol.TProtocolException; -import org.apache.thrift.protocol.TProtocolFactory; -import org.apache.thrift.transport.TTransport; - -/** - * Binary protocol implementation for thrift. - * Will not read messages bigger than 12MiB. - * - */ -public class TBinaryProtocolSafe extends TBinaryProtocol -{ - /** - * Factory - */ - @SuppressWarnings( "serial" ) - public static class Factory implements TProtocolFactory - { - - protected boolean strictRead_ = false; - protected boolean strictWrite_ = true; - - public Factory() - { - this( false, true ); - } - - public Factory(boolean strictRead, boolean strictWrite) - { - strictRead_ = strictRead; - strictWrite_ = strictWrite; - } - - public TProtocol getProtocol( TTransport trans ) - { - return new TBinaryProtocolSafe( trans, strictRead_, strictWrite_ ); - } - } - - private static final int maxLen = 12 * 1024 * 1024; // 12 MiB - - /** - * Constructor - */ - public TBinaryProtocolSafe(TTransport trans) - { - this( trans, false, true ); - } - - public TBinaryProtocolSafe(TTransport trans, boolean strictRead, boolean strictWrite) - { - super( trans ); - strictRead_ = strictRead; - strictWrite_ = strictWrite; - } - - /** - * Reading methods. - */ - - public TMessage readMessageBegin() throws TException - { - int size = readI32(); - if ( size > maxLen ) - throw new TProtocolException( TProtocolException.SIZE_LIMIT, "Payload too big." ); - if ( size < 0 ) { - int version = size & VERSION_MASK; - if ( version != VERSION_1 ) { - throw new TProtocolException( TProtocolException.BAD_VERSION, "Bad version in readMessageBegin" ); - } - return new TMessage( readString(), (byte) ( size & 0x000000ff ), readI32() ); - } else { - if ( strictRead_ ) { - throw new TProtocolException( TProtocolException.BAD_VERSION, "Missing version in readMessageBegin, old client?" ); - } - return new TMessage( readStringBody( size ), readByte(), readI32() ); - } - } - - public String readString() throws TException - { - int size = readI32(); - if ( size > maxLen ) - throw new TProtocolException( TProtocolException.SIZE_LIMIT, "Payload too big." ); - if ( trans_.getBytesRemainingInBuffer() >= size ) { - try { - String s = new String( trans_.getBuffer(), trans_.getBufferPosition(), size, "UTF-8" ); - trans_.consumeBuffer( size ); - return s; - } catch ( UnsupportedEncodingException e ) { - throw new TException( "JVM DOES NOT SUPPORT UTF-8" ); - } - } - - return readStringBody( size ); - } - - public ByteBuffer readBinary() throws TException - { - int size = readI32(); - if ( size > maxLen ) - throw new TProtocolException( TProtocolException.SIZE_LIMIT, "Payload too big." ); - if ( trans_.getBytesRemainingInBuffer() >= size ) { - ByteBuffer bb = ByteBuffer.wrap( trans_.getBuffer(), trans_.getBufferPosition(), size ); - trans_.consumeBuffer( size ); - return bb; - } - - byte[] buf = new byte[ size ]; - trans_.readAll( buf, 0, size ); - return ByteBuffer.wrap( buf ); - } - -} - diff --git a/dozentenmodulserver/src/main/java/sql/MysqlConnection.java b/dozentenmodulserver/src/main/java/sql/MysqlConnection.java deleted file mode 100644 index dbbddfe1..00000000 --- a/dozentenmodulserver/src/main/java/sql/MysqlConnection.java +++ /dev/null @@ -1,74 +0,0 @@ -package sql; - -import java.sql.Connection; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.List; - -import org.apache.log4j.Logger; - -public class MysqlConnection implements AutoCloseable { - - private static final Logger LOGGER = Logger.getLogger(MysqlConnection.class); - - private static final int CONNECTION_TIMEOUT_MS = 5 * 60 * 1000; - - private final long deadline = System.currentTimeMillis() + CONNECTION_TIMEOUT_MS; - - private final Connection rawConnection; - - private boolean hasPendingQueries = false; - - private List openStatements = new ArrayList<>(); - - MysqlConnection(Connection rawConnection) { - this.rawConnection = rawConnection; - } - - public MysqlStatement prepareStatement(String sql) throws SQLException { - hasPendingQueries = true; - MysqlStatement statement = new MysqlStatement(rawConnection, sql); - openStatements.add(statement); - return statement; - } - - public void commit() throws SQLException { - rawConnection.commit(); - hasPendingQueries = false; - } - - public void rollback() throws SQLException { - rawConnection.rollback(); - hasPendingQueries = false; - } - - boolean isValid() { - return System.currentTimeMillis() < deadline; - } - - @Override - public void close() { - if (hasPendingQueries) { - LOGGER.warn("Mysql connection had uncommited queries on .close()"); - try { - rawConnection.rollback(); - } catch (SQLException e) { - LOGGER.warn("Rolling back uncommited queries failed!", e); - } - for (MysqlStatement statement : openStatements) { - statement.close(); - } - openStatements.clear(); - } - SQL.returnConnection(this); - } - - void release() { - try { - rawConnection.close(); - } catch (SQLException e) { - // Nothing meaningful to do - } - } - -} diff --git a/dozentenmodulserver/src/main/java/sql/MysqlStatement.java b/dozentenmodulserver/src/main/java/sql/MysqlStatement.java deleted file mode 100644 index efef88b0..00000000 --- a/dozentenmodulserver/src/main/java/sql/MysqlStatement.java +++ /dev/null @@ -1,291 +0,0 @@ -package sql; - -import java.io.Closeable; -import java.sql.Connection; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.sql.Statement; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; - -/** - * Class for creating {@link java.sql.PreparedStatement}s with - * named parameters. - * Based on - * Named Parameters for PreparedStatement - */ -public class MysqlStatement implements Closeable { - - private static final QueryCache cache = new QueryCache(); - - private final PreparsedQuery query; - - private final PreparedStatement statement; - - private final List openResultSets = new ArrayList<>(); - - MysqlStatement(Connection con, String sql) throws SQLException { - PreparsedQuery query; - synchronized (cache) { - query = cache.get(sql); - } - if (query == null) { - query = parse(sql); - synchronized (cache) { - cache.put(sql, query); - } - } - this.query = query; - this.statement = con.prepareStatement(query.sql); - } - - /** - * Returns the indexes for a parameter. - * - * @param name parameter name - * @return parameter indexes - * @throws IllegalArgumentException if the parameter does not exist - */ - private List getIndexes(String name) { - List indexes = query.indexMap.get(name); - if (indexes == null) { - throw new IllegalArgumentException("Parameter not found: " + name); - } - return indexes; - } - - /** - * Sets a parameter. - * - * @param name parameter name - * @param value parameter value - * @throws SQLException if an error occurred - * @throws IllegalArgumentException if the parameter does not exist - * @see PreparedStatement#setObject(int, java.lang.Object) - */ - public void setObject(String name, Object value) throws SQLException { - List indexes = getIndexes(name); - for (Integer index : indexes) { - statement.setObject(index, value); - } - } - - /** - * Sets a parameter. - * - * @param name parameter name - * @param value parameter value - * @throws SQLException if an error occurred - * @throws IllegalArgumentException if the parameter does not exist - * @see PreparedStatement#setString(int, java.lang.String) - */ - public void setString(String name, String value) throws SQLException { - List indexes = getIndexes(name); - for (Integer index : indexes) { - statement.setString(index, value); - } - } - - /** - * Sets a parameter. - * - * @param name parameter name - * @param value parameter value - * @throws SQLException if an error occurred - * @throws IllegalArgumentException if the parameter does not exist - * @see PreparedStatement#setInt(int, int) - */ - public void setInt(String name, int value) throws SQLException { - List indexes = getIndexes(name); - for (Integer index : indexes) { - statement.setInt(index, value); - } - } - - /** - * Sets a parameter. - * - * @param name parameter name - * @param value parameter value - * @throws SQLException if an error occurred - * @throws IllegalArgumentException if the parameter does not exist - * @see PreparedStatement#setInt(int, int) - */ - public void setLong(String name, long value) throws SQLException { - List indexes = getIndexes(name); - for (Integer index : indexes) { - statement.setLong(index, value); - } - } - - /** - * Executes the statement. - * - * @return true if the first result is a {@link ResultSet} - * @throws SQLException if an error occurred - * @see PreparedStatement#execute() - */ - public boolean execute() throws SQLException { - return statement.execute(); - } - - /** - * Executes the statement, which must be a query. - * - * @return the query results - * @throws SQLException if an error occurred - * @see PreparedStatement#executeQuery() - */ - public ResultSet executeQuery() throws SQLException { - ResultSet rs = statement.executeQuery(); - openResultSets.add(rs); - return rs; - } - - /** - * Executes the statement, which must be an SQL INSERT, UPDATE or DELETE - * statement; or an SQL statement that returns nothing, such as a DDL - * statement. - * - * @return number of rows affected - * @throws SQLException if an error occurred - * @see PreparedStatement#executeUpdate() - */ - public int executeUpdate() throws SQLException { - return statement.executeUpdate(); - } - - /** - * Closes the statement. - * - * @see Statement#close() - */ - @Override - public void close() { - for (ResultSet rs : openResultSets) { - try { - rs.close(); - } catch (SQLException e) { - // - } - } - try { - statement.close(); - } catch (SQLException e) { - // Nothing to do - } - } - - /** - * Adds the current set of parameters as a batch entry. - * - * @throws SQLException if something went wrong - */ - public void addBatch() throws SQLException { - statement.addBatch(); - } - - /** - * Executes all of the batched statements. - * - * See {@link Statement#executeBatch()} for details. - * - * @return update counts for each statement - * @throws SQLException if something went wrong - */ - public int[] executeBatch() throws SQLException { - return statement.executeBatch(); - } - - // static methods - - private static PreparsedQuery parse(String query) { - int length = query.length(); - StringBuffer parsedQuery = new StringBuffer(length); - Map> paramMap = new HashMap<>(); - boolean inSingleQuote = false; - boolean inDoubleQuote = false; - boolean hasBackslash = false; - int index = 1; - - for (int i = 0; i < length; i++) { - char c = query.charAt(i); - if (hasBackslash) { - // Last char was a backslash, so we ignore the current char - hasBackslash = false; - } else if (c == '\\') { - // This is a backslash, next char will be escaped - hasBackslash = true; - } else if (inSingleQuote) { - // End of quoted string - if (c == '\'') { - inSingleQuote = false; - } - } else if (inDoubleQuote) { - // End of quoted string - if (c == '"') { - inDoubleQuote = false; - } - } else { - // Not in string, look for named params - if (c == '\'') { - inSingleQuote = true; - } else if (c == '"') { - inDoubleQuote = true; - } else if (c == ':' && i + 1 < length && Character.isJavaIdentifierStart(query.charAt(i + 1))) { - int j = i + 2; - while (j < length && Character.isJavaIdentifierPart(query.charAt(j))) { - j++; - } - String name = query.substring(i + 1, j); - c = '?'; // replace the parameter with a question mark - i += name.length(); // skip past the end of the parameter - - List indexList = paramMap.get(name); - if (indexList == null) { - indexList = new ArrayList<>(); - paramMap.put(name, indexList); - } - indexList.add(new Integer(index)); - - index++; - } - } - parsedQuery.append(c); - } - - return new PreparsedQuery(parsedQuery.toString(), paramMap); - } - - // private helper classes - - private static class PreparsedQuery { - private final Map> indexMap; - private final String sql; - - public PreparsedQuery(String sql, Map> indexMap) { - this.sql = sql; - this.indexMap = indexMap; - } - } - - private static class QueryCache extends LinkedHashMap { - private static final long serialVersionUID = 1L; - - public QueryCache() { - super(30, (float) 0.75, true); - } - - @Override - protected boolean removeEldestEntry(Map.Entry eldest) { - return size() > 40; - } - } - -} diff --git a/dozentenmodulserver/src/main/java/sql/SQL.java b/dozentenmodulserver/src/main/java/sql/SQL.java deleted file mode 100644 index af79b521..00000000 --- a/dozentenmodulserver/src/main/java/sql/SQL.java +++ /dev/null @@ -1,81 +0,0 @@ -package sql; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.util.Collections; -import java.util.Queue; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentLinkedQueue; - -import models.Configuration; - -import org.apache.log4j.Logger; - -public class SQL { - - private static final Logger LOGGER = Logger.getLogger(SQL.class); - /** - * Pool of available connections. - */ - private static final Queue pool = new ConcurrentLinkedQueue<>(); - - /** - * Set of connections currently handed out. - */ - private static final Set busyConnections = Collections.newSetFromMap(new ConcurrentHashMap()); - - static { - try { - Class.forName("com.mysql.jdbc.Driver").newInstance(); - } catch (InstantiationException | IllegalAccessException | ClassNotFoundException e) { - LOGGER.fatal("Cannot get mysql JDBC driver!", e); - System.exit(1); - } - } - - public static MysqlConnection getConnection() { - MysqlConnection con; - for (;;) { - con = pool.poll(); - if (con == null) - break; - if (!con.isValid()) { - con.release(); - continue; - } - if (!busyConnections.add(con)) - throw new RuntimeException("Tried to hand out a busy connection!"); - return con; - } - // No pooled connection - if (busyConnections.size() > 20) { - LOGGER.warn("Too many open MySQL connections. Possible connection leak!"); - return null; - } - try { - // Create fresh connection - Connection rawConnection = DriverManager.getConnection(Configuration.getDbUri(), - Configuration.getDbUsername(), Configuration.getDbPassword()); - // By convention in our program we don't want auto commit - rawConnection.setAutoCommit(false); - // Wrap into our proxy - con = new MysqlConnection(rawConnection); - // Keep track of busy mysql connection - if (!busyConnections.add(con)) - throw new RuntimeException("Tried to hand out a busy connection!"); - return con; - } catch (SQLException e) { - LOGGER.info("Failed to connect to local mysql server", e); - } - return null; - } - - static void returnConnection(MysqlConnection connection) { - if (!busyConnections.remove(connection)) - throw new RuntimeException("Tried to return a mysql connection to the pool that was not taken!"); - pool.add(connection); - } - -}// end class diff --git a/dozentenmodulserver/src/main/java/sql/models/DbImage.java b/dozentenmodulserver/src/main/java/sql/models/DbImage.java deleted file mode 100644 index fe59dac8..00000000 --- a/dozentenmodulserver/src/main/java/sql/models/DbImage.java +++ /dev/null @@ -1,41 +0,0 @@ -package sql.models; - -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.List; - -import org.apache.log4j.Logger; -import org.openslx.bwlp.thrift.iface.ImageSummaryRead; -import org.openslx.bwlp.thrift.iface.UserInfo; - -import sql.MysqlConnection; -import sql.MysqlStatement; -import sql.SQL; - -public class DbImage { - - private static final Logger LOGGER = Logger.getLogger(DbImage.class); - - public static List getAllVisible(UserInfo user, List tagSearch) { - try (MysqlConnection connection = SQL.getConnection()) { - MysqlStatement stmt = connection.prepareStatement("SELECT" - + " i.imagebaseid, i.currentversionid, i.latestversionid, i.displayname," - + " i.osid, i.virtid, i.createtime, i.updatetime, i.ownerid, i.uploaderid," - + " i.sharemode, i.istemplate, i.canlinkdefault, i.candownloaddefault," - + " i.caneditdefault, i.canadmindefault," - + " cur.filesize, cur.isenabled, cur.isrestricted, cur.isvalid," + " lat.isprocessed" - + " FROM imagebase i" - + " LEFT JOIN imageversion cur ON (cur.imageversionid = i.currentversionid)" - + " LEFT JOIN imageversion lat ON (lat.imageversionid = i.latestversionid)"); - ResultSet rs = stmt.executeQuery(); - while (rs.next()) { - ImageSummaryRead entry = new ImageSummaryRead(); - } - return null; - } catch (SQLException e) { - LOGGER.error("Query failed in DbImage.getAllVisible()", e); - return null; - } - } - -} diff --git a/dozentenmodulserver/src/main/java/thrift/OperatingSystemList.java b/dozentenmodulserver/src/main/java/thrift/OperatingSystemList.java deleted file mode 100644 index 6d5404f7..00000000 --- a/dozentenmodulserver/src/main/java/thrift/OperatingSystemList.java +++ /dev/null @@ -1,37 +0,0 @@ -package thrift; - -import java.util.List; - -import org.apache.log4j.Logger; -import org.apache.thrift.TException; -import org.openslx.bwlp.thrift.iface.OperatingSystem; -import org.openslx.thrifthelper.ThriftManager; -import org.openslx.util.TimeoutReference; - -/** - * Holds the list of all known operating systems. The list is synchronized with - * the master server. - */ -public class OperatingSystemList { - - private static final Logger LOGGER = Logger.getLogger(OperatingSystemList.class); - - private static final TimeoutReference> cachedList = new TimeoutReference>( - 600000, null); - - public static synchronized List get() { - List list = cachedList.get(); - if (list == null) { - try { - list = ThriftManager.getMasterClient().getOperatingSystems(); - // TODO: Write to DB - } catch (TException e) { - LOGGER.warn("Could not retrieve OS list from master server", e); - return null; - } - cachedList.set(list); - } - return list; - } - -} diff --git a/dozentenmodulserver/src/main/java/util/Constants.java b/dozentenmodulserver/src/main/java/util/Constants.java deleted file mode 100644 index 8ac5dabd..00000000 --- a/dozentenmodulserver/src/main/java/util/Constants.java +++ /dev/null @@ -1,21 +0,0 @@ -package util; - -import fileserv.FileChunk; - -public class Constants { - public static final String INCOMPLETE_UPLOAD_SUFFIX = ".part"; - public static final int MAX_UPLOADS; - - static { - long maxMem = Runtime.getRuntime().maxMemory(); - if (maxMem == Long.MAX_VALUE) { - // Apparently the JVM was started without a memory limit (no -Xmx cmdline), - // so we assume a default of 256MB - maxMem = 256l * 1024l * 1024l; - } - maxMem /= 1024l * 1024l; - // Now maxMem is the amount of memory in MiB - - MAX_UPLOADS = (int) Math.max(1, (maxMem - 64) / (FileChunk.CHUNK_SIZE_MIB + 1)); - } -} diff --git a/dozentenmodulserver/src/main/java/util/FileSystem.java b/dozentenmodulserver/src/main/java/util/FileSystem.java deleted file mode 100644 index 5f5a1e08..00000000 --- a/dozentenmodulserver/src/main/java/util/FileSystem.java +++ /dev/null @@ -1,25 +0,0 @@ -package util; - -import java.io.File; - -import org.apache.log4j.Logger; - -public class FileSystem { - - private static final Logger LOGGER = Logger.getLogger(FileSystem.class); - - public static String getRelativePath(File absolutePath, File parentDir) { - String file; - String dir; - try { - file = absolutePath.getCanonicalPath(); - dir = parentDir.getCanonicalPath() + File.separator; - } catch (Exception e) { - LOGGER.error("Could not get relative path for " + absolutePath.toString(), e); - return null; - } - if (!file.startsWith(dir)) - return null; - return file.substring(dir.length()); - } -} diff --git a/dozentenmodulserver/src/main/java/util/Formatter.java b/dozentenmodulserver/src/main/java/util/Formatter.java deleted file mode 100644 index 2f6fbae2..00000000 --- a/dozentenmodulserver/src/main/java/util/Formatter.java +++ /dev/null @@ -1,59 +0,0 @@ -package util; - -import java.io.File; -import java.util.UUID; - -import models.Configuration; - -import org.joda.time.format.DateTimeFormat; -import org.joda.time.format.DateTimeFormatter; -import org.openslx.bwlp.thrift.iface.UserInfo; - -public class Formatter { - - private static final DateTimeFormatter vmNameDateFormat = DateTimeFormat.forPattern("dd_HH-mm-ss"); - - /** - * Generate a unique file name used for a virtual machine - * image that is currently uploading. - * - * @return Absolute path name of file - */ - public static File getTempImageName() { - return new File(Configuration.getCurrentVmStorePath(), UUID.randomUUID().toString() - + Constants.INCOMPLETE_UPLOAD_SUFFIX); - } - - /** - * Generate a file name for the given VM based on owner and display name. - * - * @param user The user associated with the VM, e.g. the owner - * @param imageName Name of the VM - * @return File name for the VM derived from the function's input - */ - public static String vmName(UserInfo user, String imageName) { - return cleanFileName(vmNameDateFormat.print(System.currentTimeMillis()) + "_" + user.lastName + "_" - + imageName); - } - - /** - * Make sure file name contains only a subset of ascii characters and is not - * too long. - * - * @param name What we want to turn into a file name - * @return A sanitized form of name that should be safe to use as a file - * name - */ - public static String cleanFileName(String name) { - if (name == null) - return "null"; - name = name.replaceAll("[^a-zA-Z0-9_\\.\\-]+", "_"); - if (name.length() > 120) - name = name.substring(0, 120); - return name; - } - - public static String userFullName(UserInfo ui) { - return ui.firstName + " " + ui.lastName; - } -} diff --git a/dozentenmodulserver/src/main/java/util/Util.java b/dozentenmodulserver/src/main/java/util/Util.java deleted file mode 100644 index 28f522b8..00000000 --- a/dozentenmodulserver/src/main/java/util/Util.java +++ /dev/null @@ -1,45 +0,0 @@ -package util; - -import java.io.Closeable; - -public class Util { - - /** - * Try to close everything passed to this method. Never throw an exception - * if it fails, just silently continue. - * - * @param item One or more objects that are Closeable - */ - public static void safeClose(Closeable... item) { - if (item == null) - return; - for (Closeable c : item) { - if (c == null) - continue; - try { - c.close(); - } catch (Exception e) { - } - } - } - - /** - * Try to close everything passed to this method. Never throw an exception - * if it fails, just silently continue. - * - * @param item One or more objects that are AutoCloseable - */ - public static void safeClose(AutoCloseable... item) { - if (item == null) - return; - for (AutoCloseable c : item) { - if (c == null) - continue; - try { - c.close(); - } catch (Exception e) { - } - } - } - -} -- cgit v1.2.3-55-g7522