[server] Return lecture vmx/netshare/ldapfilter as .tar.gz

Save some network roundtrips and make extending delivered data much
easier. Adding another http request for every bit of information won't
scale over time.

3 files changed, 114 insertions, 69 deletions

diff --git a/dozentenmodulserver/pom.xml b/dozentenmodulserver/pom.xml
index 0766b111..f660d51d 100644
--- a/dozentenmodulserver/pom.xml
+++ b/dozentenmodulserver/pom.xml
@@ -164,6 +164,11 @@
 			<version>2.7.1-P2</version>
 			<scope>compile</scope>
 		</dependency>
+		<dependency>
+			<groupId>org.kamranzafar</groupId>
+			<artifactId>jtar</artifactId>
+			<version>2.3</version>
+		</dependency>
 	</dependencies>
 </project>
 
diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbLecture.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbLecture.java
index 57ae7778..603b72fe 100644
--- a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbLecture.java
+++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbLecture.java
@@ -279,6 +279,19 @@ public class DbLecture {
 		}
 		return list;
 	}
+	
+	private static List<NetRule> decodeNetrules(String netrules) {
+		if (netrules == null)
+			return null;
+		try {
+			NetRule[] rules = Json.deserialize(netrules, NetRule[].class);
+			if (rules != null)
+				return Arrays.asList(rules);
+		} catch (JsonParseException e) {
+			LOGGER.warn("Could not deserialize netrules", e);
+		}
+		return null;
+	}
 
 	public static LectureRead getLectureDetails(UserInfo user, String lectureId) throws SQLException,
 			TNotFoundException {
@@ -319,17 +332,7 @@ public class DbLecture {
 			lecture.setUpdaterId(rs.getString("updaterid"));
 			lecture.setRunscript(rs.getString("runscript"));
 			lecture.setNics(null); // TODO fill nics
-			String netrules = rs.getString("netrules");
-			if (netrules != null) {
-				try {
-					NetRule[] rules = Json.deserialize(netrules, NetRule[].class);
-					if (rules != null) {
-						lecture.setNetworkExceptions(Arrays.asList(rules));
-					}
-				} catch (JsonParseException e) {
-					LOGGER.warn("Could not deserialize netrules for lecture " + lectureId, e);
-				}
-			}
+			lecture.setNetworkExceptions(decodeNetrules(rs.getString("netrules")));
 			lecture.setIsExam(rs.getBoolean("isexam"));
 			lecture.setHasInternetAccess(rs.getBoolean("hasinternetaccess"));
 			lecture.setHasUsbAccess(rs.getBoolean("hasusbaccess"));
@@ -532,17 +535,19 @@ public class DbLecture {
 		}
 	}
 
-	public static VmMetaData<?, ?, ?, ?> getClientLaunchData(String lectureId) throws SQLException,
+	public static LaunchData getClientLaunchData(String lectureId) throws SQLException,
 			TNotFoundException, UnsupportedVirtualizerFormatException {
+		LaunchData retval = new LaunchData();
 		byte[] config;
 		String lectureName;
 		String osKeyword;
 		boolean usbAccess;
+		VmMetaData<?, ?, ?, ?> meta = null;
 		try (MysqlConnection connection = Database.getConnection()) {
 			// Get required data about lecture and used image
 			MysqlStatement stmt = connection.prepareStatement("SELECT"
 					+ " l.displayname AS lecturename, l.starttime, l.endtime, l.isenabled, l.hasusbaccess,"
-					+ " o.virtoskeyword, i.virtualizerconfig"
+					+ " l.runscript, o.virtoskeyword, i.virtualizerconfig"
 					+ "                      FROM lecture l                 "
 					+ " INNER JOIN imageversion i USING (imageversionid)"
 					+ " INNER JOIN imagebase b USING (imagebaseid)"
@@ -555,9 +560,21 @@ public class DbLecture {
 				throw new TNotFoundException();
 			}
 			config = rs.getBytes("virtualizerconfig");
+			if (config == null) {
+				return null;
+			}
+			try {
+				meta = VmMetaData.getInstance(OperatingSystemList.get(), config, config.length);
+			} catch (Exception e) {
+				LOGGER.error("meta could not be initialized", e);
+				return null;
+			}
 			lectureName = rs.getString("lecturename");
 			osKeyword = rs.getString("virtoskeyword");
 			usbAccess = rs.getBoolean("hasusbaccess");
+			retval.vmx = meta;
+			retval.runScript = rs.getString("runscript");
+			retval.netShares = DbLectureNetshare.getLectureNetshares(connection, lectureId);
 			// Everything worked so far, update statistics counters
 			MysqlStatement upStmt = connection.prepareStatement("UPDATE"
 					+ " lecture SET lastused = UNIX_TIMESTAMP(), usecount = usecount + 1"
@@ -569,16 +586,6 @@ public class DbLecture {
 			LOGGER.error("Query failed in DbLecture.getClientLaunchData()", e);
 			throw e;
 		}
-		if (config == null) {
-			return null;
-		}
-		VmMetaData<?, ?, ?, ?> meta = null;
-		try {
-			meta = VmMetaData.getInstance(OperatingSystemList.get(), config, config.length);
-		} catch (Exception e) {
-			LOGGER.error("meta could not be initialized", e);
-			return null;
-		}
 		meta.addDisplayName(lectureName);
 		if (osKeyword != null) {
 			meta.setOs(osKeyword);
@@ -587,7 +594,7 @@ public class DbLecture {
 		meta.addEthernet(VmMetaData.EtherType.NAT); // TODO: Use config
 		meta.enableUsb(usbAccess);
 		meta.disableSuspend();
-		return meta;
+		return retval;
 	}
 
 	public static void deleteOld(int minAgeDays) throws SQLException {
@@ -645,5 +652,11 @@ public class DbLecture {
 		uStmt.executeUpdate();
 		MailGenerator.lectureDeactivated(lectures);
 	}
+	
+	public static class LaunchData {
+		public VmMetaData<?, ?, ?, ?> vmx;
+		public List<NetShare> netShares;
+		public String runScript;
+	}
 
 }
diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebServer.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebServer.java
index 2f742420..e1823bf8 100644
--- a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebServer.java
+++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebServer.java
@@ -2,23 +2,35 @@ package org.openslx.bwlp.sat.web;
 
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
+import java.io.PipedInputStream;
+import java.io.PipedOutputStream;
+import java.nio.charset.StandardCharsets;
 import java.sql.SQLException;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.zip.GZIPOutputStream;
 
 import org.apache.commons.io.output.ByteArrayOutputStream;
 import org.apache.log4j.Logger;
+import org.kamranzafar.jtar.TarEntry;
+import org.kamranzafar.jtar.TarHeader;
+import org.kamranzafar.jtar.TarOutputStream;
 import org.openslx.bwlp.sat.database.mappers.DbLecture;
-import org.openslx.bwlp.sat.database.mappers.DbLectureNetshare;
+import org.openslx.bwlp.sat.database.mappers.DbLecture.LaunchData;
 import org.openslx.bwlp.sat.fileserv.FileServer;
 import org.openslx.bwlp.thrift.iface.LectureRead;
 import org.openslx.bwlp.thrift.iface.NetRule;
 import org.openslx.bwlp.thrift.iface.NetShare;
 import org.openslx.bwlp.thrift.iface.NetShareAuth;
 import org.openslx.bwlp.thrift.iface.TNotFoundException;
+import org.openslx.util.GrowingThreadPoolExecutor;
 import org.openslx.util.Json;
+import org.openslx.util.Util;
 import org.openslx.util.vm.UnsupportedVirtualizerFormatException;
-import org.openslx.util.vm.VmMetaData;
 import org.simpleframework.xml.Serializer;
 import org.simpleframework.xml.core.Persister;
 
@@ -27,6 +39,9 @@ import fi.iki.elonen.NanoHTTPD;
 public class WebServer extends NanoHTTPD {
 
 	private static final Logger LOGGER = Logger.getLogger(WebServer.class);
+	
+	private static final ThreadPoolExecutor tpe = 
+			new GrowingThreadPoolExecutor(1, 8, 1, TimeUnit.MINUTES, new LinkedBlockingQueue<Runnable>(16));
 
 	private static final Serializer serializer = new Persister();
 
@@ -93,16 +108,12 @@ public class WebServer extends NanoHTTPD {
 				}
 			}
 			if (parts[1].equals("lecture")) {
-				if (parts.length < 3)
+				if (parts.length < 4)
 					return badRequest("Bad Request");
-				if (parts.length < 4 || parts[3].equals("vmx"))
-					return serveLectureStart(parts[2]);
+				if (parts[3].equals("metadata"))
+					return serveMetaData(parts[2]);
 				if (parts[3].equals("netrules"))
 					return serveLectureNetRules(parts[2]);
-				if (parts[3].equals("netshares"))
-					return serveLectureNetshares(parts[2]);
-				if (parts[3].equals("runscript"))
-					return serveLectureScript(parts[2]);
 			}
 			return notFound();
 		}
@@ -126,6 +137,21 @@ public class WebServer extends NanoHTTPD {
 		return new NanoHTTPD.Response(NanoHTTPD.Response.Status.OK, "application/json; charset=utf-8",
 				Json.serialize(FileServer.instance().getStatus()));
 	}
+	
+	private static void tarPutFile(TarOutputStream output, String fileName, String data) throws IOException
+	{
+		if (data == null)
+			return;
+		tarPutFile(output, fileName, data.getBytes(StandardCharsets.UTF_8));
+	}
+	
+	private static void tarPutFile(TarOutputStream output, String fileName, byte[] data) throws IOException
+	{
+		if (data == null)
+			return;
+		output.putNextEntry(new TarEntry(TarHeader.createHeader(fileName, data.length, Util.unixTime(), false, 0644)));
+		output.write(data);
+	}
 
 	/**
 	 * Return meta data (eg. *.vmx) required to start the given lecture.
@@ -133,20 +159,45 @@ public class WebServer extends NanoHTTPD {
 	 * @param lectureId
 	 * @return
 	 */
-	private Response serveLectureStart(String lectureId) {
-		VmMetaData<?, ?, ?, ?> meta;
+	private Response serveMetaData(final String lectureId) {
+		PipedInputStream sink = new PipedInputStream(10000);
 		try {
-			meta = DbLecture.getClientLaunchData(lectureId);
-		} catch (UnsupportedVirtualizerFormatException | TNotFoundException e) {
-			// TODO better virt error handling
-			return notFound();
-		} catch (SQLException e) {
+			final TarOutputStream output = new TarOutputStream(new GZIPOutputStream(new PipedOutputStream(sink)));
+			final LaunchData ld;
+			try {
+				ld = DbLecture.getClientLaunchData(lectureId);
+			} catch (UnsupportedVirtualizerFormatException | TNotFoundException e) {
+				// TODO better virt error handling
+				return notFound();
+			} catch (SQLException e) {
+				return internalServerError();
+			}
+			// Meta is required, everything else is optional
+			tpe.execute(new Runnable() {
+				@Override
+				public void run() {
+					try {
+						tarPutFile(output, "vmx", ld.vmx.getFilteredDefinitionArray());
+						tarPutFile(output, "runscript", ld.runScript);
+						tarPutFile(output, "netshares", serializeNetShares(ld.netShares));
+					} catch (IOException e) {
+						LOGGER.warn("Error writing to tar stream", e);
+					} finally {
+						Util.safeClose(output);
+					}
+				}
+			});
+		} catch (IOException e1) {
+			LOGGER.warn("Could not create tar output stream", e1);
+			return internalServerError();
+		} catch (RejectedExecutionException e2) {
+			LOGGER.warn("Server overloaded; rejecting VM Metadata request", e2);
 			return internalServerError();
 		}
-		return new NanoHTTPD.Response(NanoHTTPD.Response.Status.OK, "text/plain; charset=utf-8",
-				new ByteArrayInputStream(meta.getFilteredDefinitionArray()));
+		return new NanoHTTPD.Response(NanoHTTPD.Response.Status.OK, "application/gzip",
+				sink);
 	}
-
+	
 	private Response serveLectureNetRules(String lectureId) {
 		LectureRead lecture;
 		try {
@@ -180,30 +231,7 @@ public class WebServer extends NanoHTTPD {
 				sb.toString());
 	}
 
-	private Response serveLectureScript(String lectureId) {
-		LectureRead lecture;
-		try {
-			lecture = DbLecture.getLectureDetails(null, lectureId);
-		} catch (TNotFoundException e) {
-			return notFound();
-		} catch (SQLException e) {
-			return internalServerError();
-		}
-		if (lecture.runscript == null) {
-			lecture.runscript = "";
-		}
-		return new NanoHTTPD.Response(NanoHTTPD.Response.Status.OK, "text/plain; charset=utf-8",
-				lecture.runscript);
-	}
-
-	private Response serveLectureNetshares(String lectureId) {
-		List<NetShare> list = null;
-		try {
-			list = DbLectureNetshare.getLectureNetshares(lectureId);
-			// TODO handling not found necessary?
-		} catch (SQLException e) {
-			return internalServerError();
-		}
+	private String serializeNetShares(List<NetShare> list) {
 		// openslx.exe expects shares in the following format
 		//	<path> <letter> <shortcut> <username> <password>
 		// letter is either a drive letter for Windows VMs,
@@ -230,8 +258,7 @@ public class WebServer extends NanoHTTPD {
 				sb.append("\n");
 			}
 		}
-		return new NanoHTTPD.Response(NanoHTTPD.Response.Status.OK, "text/plain; charset=utf-8",
-				sb.toString());
+		return sb.toString();
 	}
 
 	/**