Add container models, which keeps the necessary informations about container imags, its meta information and for lecture specific information

6 files changed, 497 insertions, 0 deletions

diff --git a/pom.xml b/pom.xml
index 7fdb164..d9daa34 100644
--- a/pom.xml
+++ b/pom.xml
@@ -170,5 +170,11 @@
 			<artifactId>jing</artifactId>
 			<version>20181222</version>
 		</dependency>
+		<dependency>
+			<groupId>org.kamranzafar</groupId>
+			<artifactId>jtar</artifactId>
+			<version>2.3</version>
+			<scope>compile</scope>
+		</dependency>
 	</dependencies>
 </project>
diff --git a/src/main/java/org/openslx/util/TarArchiveUtil.java b/src/main/java/org/openslx/util/TarArchiveUtil.java
new file mode 100644
index 0000000..09c4a2c
--- /dev/null
+++ b/src/main/java/org/openslx/util/TarArchiveUtil.java
@@ -0,0 +1,29 @@
+package org.openslx.util;
+
+import org.kamranzafar.jtar.TarEntry;
+import org.kamranzafar.jtar.TarHeader;
+import org.kamranzafar.jtar.TarOutputStream;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+
+public class TarArchiveUtil {
+
+
+
+	public static void tarPutFile(TarOutputStream output, String fileName, String data) throws IOException
+	{
+		if (data == null)
+			return;
+		tarPutFile(output, fileName, data.getBytes(StandardCharsets.UTF_8));
+	}
+
+	public static void tarPutFile(TarOutputStream output, String fileName, byte[] data) throws IOException
+	{
+		if (data == null)
+			return;
+		output.putNextEntry(new TarEntry(
+				TarHeader.createHeader(fileName, data.length, Util.unixTime(), false, 0644)));
+		output.write(data);
+	}
+}
diff --git a/src/main/java/org/openslx/virtualization/configuration/container/ContainerBindMount.java b/src/main/java/org/openslx/virtualization/configuration/container/ContainerBindMount.java
new file mode 100644
index 0000000..ca8f35f
--- /dev/null
+++ b/src/main/java/org/openslx/virtualization/configuration/container/ContainerBindMount.java
@@ -0,0 +1,62 @@
+package org.openslx.virtualization.configuration.container;
+
+import java.util.Objects;
+
+/**
+ * This class implements a model for a bind mount entry in the docker context
+ * (eg. docker run  ... --mount type=bind,source=source,target=target,options ... ). A list of objects of this class is stored in
+ * {@link ContainerMeta}.
+ */
+public class ContainerBindMount {
+
+	private String source;
+	private String target;
+	private String options;
+
+	public ContainerBindMount() {
+	}
+
+	public ContainerBindMount(String source, String target, String options) {
+		this.source = source;
+		this.target = target;
+		this.options = options;
+	}
+
+	public String getSource() {
+		return source;
+	}
+
+	public void setSource(String source) {
+		this.source = source;
+	}
+
+	public String getTarget() {
+		return target;
+	}
+
+	public void setTarget(String target) {
+		this.target = target;
+	}
+
+	public String getOptions() {
+		return options;
+	}
+
+	public void setOptions(String options) {
+		this.options = options;
+	}
+
+	@Override public boolean equals(Object o) {
+		if (this == o)
+			return true;
+		if (o == null || getClass() != o.getClass())
+			return false;
+		ContainerBindMount that = (ContainerBindMount) o;
+		return Objects.equals(source, that.source) && Objects.equals(target, that.target) && Objects.equals(
+				options, that.options);
+	}
+
+	@Override public int hashCode() {
+		return Objects.hash(source, target, options);
+	}
+}
diff --git a/src/main/java/org/openslx/virtualization/configuration/container/ContainerBuildContextMethod.java b/src/main/java/org/openslx/virtualization/configuration/container/ContainerBuildContextMethod.java
new file mode 100644
index 0000000..760dc61
--- /dev/null
+++ b/src/main/java/org/openslx/virtualization/configuration/container/ContainerBuildContextMethod.java
@@ -0,0 +1,10 @@
+package org.openslx.virtualization.configuration.container;
+
+public enum ContainerBuildContextMethod {
+
+	FILE, GIT_REPOSITORY,IMAGE_REPO;
+
+	public static ContainerBuildContextMethod fromInt(int index) {
+		return values()[index];
+	}
+}
diff --git a/src/main/java/org/openslx/virtualization/configuration/container/ContainerDefinition.java b/src/main/java/org/openslx/virtualization/configuration/container/ContainerDefinition.java
new file mode 100644
index 0000000..ef493f9
--- /dev/null
+++ b/src/main/java/org/openslx/virtualization/configuration/container/ContainerDefinition.java
@@ -0,0 +1,225 @@
+package org.openslx.virtualization.configuration.container;
+
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.stream.JsonReader;
+import org.apache.log4j.Logger;
+import org.kamranzafar.jtar.TarEntry;
+import org.kamranzafar.jtar.TarInputStream;
+import org.kamranzafar.jtar.TarOutputStream;
+import org.openslx.util.TarArchiveUtil;
+
+import java.io.*;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.Objects;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.GZIPOutputStream;
+
+public class ContainerDefinition {
+
+	//	TODO database needs a refactoring to store container details
+	//  TODO tar.gz of this object is not useful, for smaller dockerfiles it makes the package lager.
+
+
+	protected static final Logger LOGGER = Logger.getLogger(ContainerDefinition.class);
+
+	protected static final String CONTAINER_FILE = "dockerfile";
+	protected static final String CONTAINER_META_FILE = "container_meta.json";
+
+	/**
+	 * The file to construct a real container image, could be an dockerfile or a singularity recipe.
+	 */
+	public String containerRecipe = "";
+
+	/**
+	 * Further container information, see {@link ContainerMeta}.
+	 */
+	public ContainerMeta containerMeta;
+
+	public ContainerDefinition() {
+		containerMeta = new ContainerMeta();
+	}
+
+	/**
+	 * Copy Constructor
+	 *
+	 * @param containerDef {@link ContainerDefinition} from which to make a deep copy.
+	 */
+	public ContainerDefinition(ContainerDefinition containerDef) {
+		containerRecipe = String.valueOf(containerDef.getContainerRecipe());
+		containerMeta = new ContainerMeta(containerDef.getContainerMeta());
+	}
+
+	public String getContainerRecipe() {
+		return containerRecipe;
+	}
+
+	public void setContainerRecipe(String containerRecipe) {
+		this.containerRecipe = containerRecipe;
+	}
+
+	public void setContainerRecipe(File containerRecipeFile) {
+		this.containerRecipe = readContainerRecipe(containerRecipeFile);
+	}
+
+	public void setContainerRecipe(byte[] rawContainerRecipe) {
+		this.containerRecipe = new String(rawContainerRecipe, StandardCharsets.UTF_8);
+	}
+
+	public void setContainerMeta(byte[] containerMeta) {
+		Gson gson = new GsonBuilder().create();
+		this.containerMeta = gson.fromJson(new JsonReader(
+						new InputStreamReader(new ByteArrayInputStream(containerMeta), StandardCharsets.UTF_8)),
+				ContainerMeta.class);
+	}
+
+	public ContainerMeta getContainerMeta() {
+		return containerMeta;
+	}
+
+//	public VirtualizationConfigurationDocker createVirtualizationConfig() {
+//		byte[] rawContainerRecipe = toByteBuffer().array();
+//		try {
+//			return new VirtualizationConfigurationDocker(MetaDataCache.getOperatingSystems(), rawContainerRecipe,
+//					rawContainerRecipe.length);
+//		} catch (VirtualizationConfigurationException e) {
+//			e.printStackTrace();
+//			LOGGER.error("Could not create ContainerMetaDataDummy");
+//		}
+//		return null;
+//	}
+
+	/**
+	 * Utility function to create a {@link ContainerDefinition} object for a byte array downloaded from the server.
+	 *
+	 * @param rawTarData Downloaded tar.gz file from the server as a byte array.
+	 * @return New object of ContainerDefinition.
+	 */
+	public static ContainerDefinition fromByteArray(byte[] rawTarData) {
+
+		ContainerDefinition containerDef = new ContainerDefinition();
+
+		try {
+			TarInputStream tis = new TarInputStream(
+					new GZIPInputStream(new BufferedInputStream(new ByteArrayInputStream(rawTarData))));
+
+			TarEntry entry;
+
+			while ((entry = tis.getNextEntry()) != null) {
+				byte[] rawData = new byte[1024];
+				ByteArrayOutputStream output = new ByteArrayOutputStream();
+				int count;
+
+				// read everything from the TarInputStream for the current Entry
+				while ((count = tis.read(rawData)) != -1) {
+					output.write(rawData, 0, count);
+				}
+
+				if (entry.getName().equals(CONTAINER_FILE))
+					containerDef.setContainerRecipe(output.toByteArray());
+				if (entry.getName().equals(CONTAINER_META_FILE))
+					containerDef.setContainerMeta(output.toByteArray());
+			}
+
+		} catch (IOException e) {
+			LOGGER.error("Could not create a ContainerDefinition Object for rawTarData", e);
+		}
+
+		return containerDef;
+	}
+
+	/**
+	 * Serializes the ContainerMeta and Container Description (e.g. dockerfile) into an tar.gz archive.
+	 *
+	 * @return A ByteBuffer object of the container definition. Can be uploaded so satellite server.
+	 */
+	public ByteBuffer toByteBuffer() {
+
+		ByteBuffer containerDef = null;
+
+		try {
+			ByteArrayOutputStream baos = new ByteArrayOutputStream();
+			TarOutputStream output = new TarOutputStream(new GZIPOutputStream(baos));
+
+			Gson gson = new GsonBuilder().setPrettyPrinting().create();
+			TarArchiveUtil.tarPutFile(output, CONTAINER_META_FILE, gson.toJson(containerMeta));
+			TarArchiveUtil.tarPutFile(output, CONTAINER_FILE, containerRecipe);
+			output.close();
+
+			containerDef = ByteBuffer.wrap(baos.toByteArray());
+
+		} catch (IOException e) {
+			LOGGER.warn("Could not create a tar file", e);
+		}
+
+		return containerDef;
+	}
+
+	private String readContainerRecipe(File file) {
+		String recipe = null;
+		try {
+
+			BufferedInputStream bis = new BufferedInputStream(new FileInputStream(file));
+			ByteArrayOutputStream rawFile = new ByteArrayOutputStream();
+			int count;
+			byte[] data = new byte[1024];
+			while ((count = bis.read(data)) != -1) {
+				rawFile.write(data, 0, count);
+			}
+
+			String rawRecipe = new String(rawFile.toByteArray(), StandardCharsets.UTF_8);
+
+			// replace windows by unix EOL
+			recipe = rawRecipe.replaceAll("\\r\\n", "\n");
+			
+			bis.close();
+
+		} catch (IOException e) {
+			LOGGER.error("Could not read Container Recipe", e);
+		}
+		return recipe;
+	}
+
+	/**
+	 * Saves containerRecipe and containerMeta at the provided location.
+	 *
+	 * @param destDir destination directory for containerRecipe and containerMeta.
+	 */
+	public void saveLocal(File destDir) {
+
+		writeFile(destDir, containerRecipe, CONTAINER_FILE);
+		Gson gson = new GsonBuilder().setPrettyPrinting().create();
+		writeFile(destDir, gson.toJson(containerMeta), CONTAINER_META_FILE);
+	}
+
+	private void writeFile(File destDir, String fileContent, String filename) {
+		File output = new File(destDir, filename);
+		try {
+			FileWriter fw = new FileWriter(output);
+			fw.write(fileContent);
+			fw.flush();
+			fw.close();
+		} catch (IOException e) {
+			e.printStackTrace();
+			LOGGER.error("Could not write File", e);
+		}
+	}
+
+	public ContainerBuildContextMethod getBuildContextMethod() {
+		return ContainerBuildContextMethod.fromInt(containerMeta.getBuildContextMethod());
+	}
+
+	@Override public boolean equals(Object o) {
+		if (this == o)
+			return true;
+		if (o == null || getClass() != o.getClass())
+			return false;
+		ContainerDefinition that = (ContainerDefinition) o;
+		return containerRecipe.equals(that.containerRecipe) && containerMeta.equals(that.containerMeta);
+	}
+
+	@Override public int hashCode() {
+		return Objects.hash(containerRecipe, containerMeta);
+	}
+}
diff --git a/src/main/java/org/openslx/virtualization/configuration/container/ContainerMeta.java b/src/main/java/org/openslx/virtualization/configuration/container/ContainerMeta.java
new file mode 100644
index 0000000..5d18248
--- /dev/null
+++ b/src/main/java/org/openslx/virtualization/configuration/container/ContainerMeta.java
@@ -0,0 +1,165 @@
+package org.openslx.virtualization.configuration.container;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+
+/**
+ * ContainerMeta is used to store container specific information.
+ * An object of this class will be serialized with gson to a json file.
+ * <p>
+ * TODO remove build_context_method
+ * no need to distinguish between methods
+ * TODO rename build_context_url to build_context
+ */
+public class ContainerMeta {
+
+	public enum ContainerImageType implements org.apache.thrift.TEnum {
+		LECTURE("lecture"), BATCH("batch"), DATA("data");
+
+		private final String name;
+
+		ContainerImageType(String name) {
+			this.name = name;
+		}
+
+		public boolean equalNames(String other) {
+			return name.equals(other);
+		}
+
+		public static ContainerImageType getByName(String name) {
+			for (ContainerImageType item : ContainerImageType.values()) {
+				if (item.name.equals(name))
+					return item;
+			}
+			// name is not an enum, return lecture as default
+			return LECTURE;
+		}
+
+		@Override public String toString() {
+			return this.name;
+		}
+
+		@Override public int getValue() {
+			return this.ordinal();
+		}
+	}
+
+	private int build_context_method;
+	private String image_repo;
+	private String build_context_url;
+	private String image_name;
+	private String run_options;
+	private String run_command;
+	private String image_type;
+	private List<ContainerBindMount> bind_mount_config = new ArrayList<>();
+
+	public ContainerMeta() {
+
+		image_repo = "";
+		build_context_method = ContainerBuildContextMethod.FILE.ordinal();
+		build_context_url = "";
+		image_name = "";
+		run_options = "";
+		run_command = "";
+		image_type = ContainerImageType.LECTURE.toString();
+		bind_mount_config = new ArrayList<>();
+	}
+
+	public ContainerMeta(ContainerMeta containerMeta) {
+		build_context_method = containerMeta.build_context_method;
+		build_context_url = containerMeta.build_context_url;
+		image_name = containerMeta.image_name;
+		run_options = containerMeta.run_options;
+		run_command = containerMeta.run_command;
+		image_repo = containerMeta.image_repo;
+
+		for (ContainerBindMount bm : containerMeta.bind_mount_config)
+			bind_mount_config.add(new ContainerBindMount(bm.getSource(), bm.getTarget(), bm.getOptions()));
+
+	}
+
+	public int getBuildContextMethod() {
+		return build_context_method;
+	}
+
+	public void setBuildContextMethod(int buildContextMethod) {
+		this.build_context_method = buildContextMethod;
+	}
+
+	public String getBuildContextUrl() {
+		return build_context_url;
+	}
+
+	public void setBuildContextUrl(String buildContextUrl) {
+		this.build_context_url = buildContextUrl;
+	}
+
+	public String getRunOptions() {
+		return run_options;
+	}
+
+	public void setRunOptions(String run_options) {
+		this.run_options = run_options;
+	}
+
+	public void setRunCommand(String run_command) {
+		this.run_command = run_command;
+	}
+
+	public String getRunCommand() {
+		return this.run_command;
+	}
+
+	public String getImageName() {
+		return image_name;
+	}
+
+	public void setImageName(String image_name) {
+		this.image_name = image_name;
+	}
+
+	public List<ContainerBindMount> getBindMountConfig() {
+		return bind_mount_config;
+	}
+
+	public void setBindMountConfig(List<ContainerBindMount> bindMountConfig) {
+		this.bind_mount_config = bindMountConfig;
+	}
+
+	public String getImageRepo() {
+		return image_repo;
+	}
+
+	public void setImageRepo(String from_image) {
+		this.image_repo = from_image;
+	}
+
+	public ContainerImageType getImageType() {
+		if (image_type == null || image_type.length() == 0)
+			return ContainerImageType.LECTURE;
+
+		return ContainerImageType.getByName(image_type);
+	}
+
+	public void setImageType(ContainerImageType image_type) {
+		this.image_type = image_type.toString();
+	}
+
+	@Override public boolean equals(Object o) {
+		if (this == o)
+			return true;
+		if (o == null || getClass() != o.getClass())
+			return false;
+		ContainerMeta that = (ContainerMeta) o;
+		return Objects.equals(build_context_url, that.build_context_url) && Objects.equals(image_name,
+				that.image_name) && Objects.equals(run_options, that.run_options) && Objects.equals(
+				run_command, that.run_command) && Objects.equals(bind_mount_config, that.bind_mount_config)
+				&& Objects.equals(image_repo, that.image_repo) && Objects.equals(image_type, that.image_type);
+	}
+
+	@Override public int hashCode() {
+		return Objects.hash(build_context_url, image_name, run_options, run_command, bind_mount_config,
+				image_repo, image_type);
+	}
+}