From b15889038b640f240555b1bab26997aa068c263b Mon Sep 17 00:00:00 2001 From: ralph isenmann Date: Wed, 23 Jun 2021 14:45:25 +0200 Subject: Add container models, which keeps the necessary informations about container imags, its meta information and for lecture specific information --- .../container/ContainerBindMount.java | 62 ++++++ .../container/ContainerBuildContextMethod.java | 10 + .../container/ContainerDefinition.java | 225 +++++++++++++++++++++ .../configuration/container/ContainerMeta.java | 165 +++++++++++++++ 4 files changed, 462 insertions(+) create mode 100644 src/main/java/org/openslx/virtualization/configuration/container/ContainerBindMount.java create mode 100644 src/main/java/org/openslx/virtualization/configuration/container/ContainerBuildContextMethod.java create mode 100644 src/main/java/org/openslx/virtualization/configuration/container/ContainerDefinition.java create mode 100644 src/main/java/org/openslx/virtualization/configuration/container/ContainerMeta.java (limited to 'src/main/java/org/openslx/virtualization/configuration/container') diff --git a/src/main/java/org/openslx/virtualization/configuration/container/ContainerBindMount.java b/src/main/java/org/openslx/virtualization/configuration/container/ContainerBindMount.java new file mode 100644 index 0000000..ca8f35f --- /dev/null +++ b/src/main/java/org/openslx/virtualization/configuration/container/ContainerBindMount.java @@ -0,0 +1,62 @@ +package org.openslx.virtualization.configuration.container; + +import java.util.Objects; + +/** + * This class implements a model for a bind mount entry in the docker context + * (eg. docker run ... --mount type=bind,source=source,target=target,options ... ). A list of objects of this class is stored in + * {@link ContainerMeta}. + */ +public class ContainerBindMount { + + private String source; + private String target; + private String options; + + public ContainerBindMount() { + } + + public ContainerBindMount(String source, String target, String options) { + this.source = source; + this.target = target; + this.options = options; + } + + public String getSource() { + return source; + } + + public void setSource(String source) { + this.source = source; + } + + public String getTarget() { + return target; + } + + public void setTarget(String target) { + this.target = target; + } + + public String getOptions() { + return options; + } + + public void setOptions(String options) { + this.options = options; + } + + @Override public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + ContainerBindMount that = (ContainerBindMount) o; + return Objects.equals(source, that.source) && Objects.equals(target, that.target) && Objects.equals( + options, that.options); + } + + @Override public int hashCode() { + return Objects.hash(source, target, options); + } +} diff --git a/src/main/java/org/openslx/virtualization/configuration/container/ContainerBuildContextMethod.java b/src/main/java/org/openslx/virtualization/configuration/container/ContainerBuildContextMethod.java new file mode 100644 index 0000000..760dc61 --- /dev/null +++ b/src/main/java/org/openslx/virtualization/configuration/container/ContainerBuildContextMethod.java @@ -0,0 +1,10 @@ +package org.openslx.virtualization.configuration.container; + +public enum ContainerBuildContextMethod { + + FILE, GIT_REPOSITORY,IMAGE_REPO; + + public static ContainerBuildContextMethod fromInt(int index) { + return values()[index]; + } +} diff --git a/src/main/java/org/openslx/virtualization/configuration/container/ContainerDefinition.java b/src/main/java/org/openslx/virtualization/configuration/container/ContainerDefinition.java new file mode 100644 index 0000000..ef493f9 --- /dev/null +++ b/src/main/java/org/openslx/virtualization/configuration/container/ContainerDefinition.java @@ -0,0 +1,225 @@ +package org.openslx.virtualization.configuration.container; + +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.stream.JsonReader; +import org.apache.log4j.Logger; +import org.kamranzafar.jtar.TarEntry; +import org.kamranzafar.jtar.TarInputStream; +import org.kamranzafar.jtar.TarOutputStream; +import org.openslx.util.TarArchiveUtil; + +import java.io.*; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.util.Objects; +import java.util.zip.GZIPInputStream; +import java.util.zip.GZIPOutputStream; + +public class ContainerDefinition { + + // TODO database needs a refactoring to store container details + // TODO tar.gz of this object is not useful, for smaller dockerfiles it makes the package lager. + + + protected static final Logger LOGGER = Logger.getLogger(ContainerDefinition.class); + + protected static final String CONTAINER_FILE = "dockerfile"; + protected static final String CONTAINER_META_FILE = "container_meta.json"; + + /** + * The file to construct a real container image, could be an dockerfile or a singularity recipe. + */ + public String containerRecipe = ""; + + /** + * Further container information, see {@link ContainerMeta}. + */ + public ContainerMeta containerMeta; + + public ContainerDefinition() { + containerMeta = new ContainerMeta(); + } + + /** + * Copy Constructor + * + * @param containerDef {@link ContainerDefinition} from which to make a deep copy. + */ + public ContainerDefinition(ContainerDefinition containerDef) { + containerRecipe = String.valueOf(containerDef.getContainerRecipe()); + containerMeta = new ContainerMeta(containerDef.getContainerMeta()); + } + + public String getContainerRecipe() { + return containerRecipe; + } + + public void setContainerRecipe(String containerRecipe) { + this.containerRecipe = containerRecipe; + } + + public void setContainerRecipe(File containerRecipeFile) { + this.containerRecipe = readContainerRecipe(containerRecipeFile); + } + + public void setContainerRecipe(byte[] rawContainerRecipe) { + this.containerRecipe = new String(rawContainerRecipe, StandardCharsets.UTF_8); + } + + public void setContainerMeta(byte[] containerMeta) { + Gson gson = new GsonBuilder().create(); + this.containerMeta = gson.fromJson(new JsonReader( + new InputStreamReader(new ByteArrayInputStream(containerMeta), StandardCharsets.UTF_8)), + ContainerMeta.class); + } + + public ContainerMeta getContainerMeta() { + return containerMeta; + } + +// public VirtualizationConfigurationDocker createVirtualizationConfig() { +// byte[] rawContainerRecipe = toByteBuffer().array(); +// try { +// return new VirtualizationConfigurationDocker(MetaDataCache.getOperatingSystems(), rawContainerRecipe, +// rawContainerRecipe.length); +// } catch (VirtualizationConfigurationException e) { +// e.printStackTrace(); +// LOGGER.error("Could not create ContainerMetaDataDummy"); +// } +// return null; +// } + + /** + * Utility function to create a {@link ContainerDefinition} object for a byte array downloaded from the server. + * + * @param rawTarData Downloaded tar.gz file from the server as a byte array. + * @return New object of ContainerDefinition. + */ + public static ContainerDefinition fromByteArray(byte[] rawTarData) { + + ContainerDefinition containerDef = new ContainerDefinition(); + + try { + TarInputStream tis = new TarInputStream( + new GZIPInputStream(new BufferedInputStream(new ByteArrayInputStream(rawTarData)))); + + TarEntry entry; + + while ((entry = tis.getNextEntry()) != null) { + byte[] rawData = new byte[1024]; + ByteArrayOutputStream output = new ByteArrayOutputStream(); + int count; + + // read everything from the TarInputStream for the current Entry + while ((count = tis.read(rawData)) != -1) { + output.write(rawData, 0, count); + } + + if (entry.getName().equals(CONTAINER_FILE)) + containerDef.setContainerRecipe(output.toByteArray()); + if (entry.getName().equals(CONTAINER_META_FILE)) + containerDef.setContainerMeta(output.toByteArray()); + } + + } catch (IOException e) { + LOGGER.error("Could not create a ContainerDefinition Object for rawTarData", e); + } + + return containerDef; + } + + /** + * Serializes the ContainerMeta and Container Description (e.g. dockerfile) into an tar.gz archive. + * + * @return A ByteBuffer object of the container definition. Can be uploaded so satellite server. + */ + public ByteBuffer toByteBuffer() { + + ByteBuffer containerDef = null; + + try { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + TarOutputStream output = new TarOutputStream(new GZIPOutputStream(baos)); + + Gson gson = new GsonBuilder().setPrettyPrinting().create(); + TarArchiveUtil.tarPutFile(output, CONTAINER_META_FILE, gson.toJson(containerMeta)); + TarArchiveUtil.tarPutFile(output, CONTAINER_FILE, containerRecipe); + output.close(); + + containerDef = ByteBuffer.wrap(baos.toByteArray()); + + } catch (IOException e) { + LOGGER.warn("Could not create a tar file", e); + } + + return containerDef; + } + + private String readContainerRecipe(File file) { + String recipe = null; + try { + + BufferedInputStream bis = new BufferedInputStream(new FileInputStream(file)); + ByteArrayOutputStream rawFile = new ByteArrayOutputStream(); + int count; + byte[] data = new byte[1024]; + while ((count = bis.read(data)) != -1) { + rawFile.write(data, 0, count); + } + + String rawRecipe = new String(rawFile.toByteArray(), StandardCharsets.UTF_8); + + // replace windows by unix EOL + recipe = rawRecipe.replaceAll("\\r\\n", "\n"); + + bis.close(); + + } catch (IOException e) { + LOGGER.error("Could not read Container Recipe", e); + } + return recipe; + } + + /** + * Saves containerRecipe and containerMeta at the provided location. + * + * @param destDir destination directory for containerRecipe and containerMeta. + */ + public void saveLocal(File destDir) { + + writeFile(destDir, containerRecipe, CONTAINER_FILE); + Gson gson = new GsonBuilder().setPrettyPrinting().create(); + writeFile(destDir, gson.toJson(containerMeta), CONTAINER_META_FILE); + } + + private void writeFile(File destDir, String fileContent, String filename) { + File output = new File(destDir, filename); + try { + FileWriter fw = new FileWriter(output); + fw.write(fileContent); + fw.flush(); + fw.close(); + } catch (IOException e) { + e.printStackTrace(); + LOGGER.error("Could not write File", e); + } + } + + public ContainerBuildContextMethod getBuildContextMethod() { + return ContainerBuildContextMethod.fromInt(containerMeta.getBuildContextMethod()); + } + + @Override public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + ContainerDefinition that = (ContainerDefinition) o; + return containerRecipe.equals(that.containerRecipe) && containerMeta.equals(that.containerMeta); + } + + @Override public int hashCode() { + return Objects.hash(containerRecipe, containerMeta); + } +} diff --git a/src/main/java/org/openslx/virtualization/configuration/container/ContainerMeta.java b/src/main/java/org/openslx/virtualization/configuration/container/ContainerMeta.java new file mode 100644 index 0000000..5d18248 --- /dev/null +++ b/src/main/java/org/openslx/virtualization/configuration/container/ContainerMeta.java @@ -0,0 +1,165 @@ +package org.openslx.virtualization.configuration.container; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +/** + * ContainerMeta is used to store container specific information. + * An object of this class will be serialized with gson to a json file. + *

+ * TODO remove build_context_method + * no need to distinguish between methods + * TODO rename build_context_url to build_context + */ +public class ContainerMeta { + + public enum ContainerImageType implements org.apache.thrift.TEnum { + LECTURE("lecture"), BATCH("batch"), DATA("data"); + + private final String name; + + ContainerImageType(String name) { + this.name = name; + } + + public boolean equalNames(String other) { + return name.equals(other); + } + + public static ContainerImageType getByName(String name) { + for (ContainerImageType item : ContainerImageType.values()) { + if (item.name.equals(name)) + return item; + } + // name is not an enum, return lecture as default + return LECTURE; + } + + @Override public String toString() { + return this.name; + } + + @Override public int getValue() { + return this.ordinal(); + } + } + + private int build_context_method; + private String image_repo; + private String build_context_url; + private String image_name; + private String run_options; + private String run_command; + private String image_type; + private List bind_mount_config = new ArrayList<>(); + + public ContainerMeta() { + + image_repo = ""; + build_context_method = ContainerBuildContextMethod.FILE.ordinal(); + build_context_url = ""; + image_name = ""; + run_options = ""; + run_command = ""; + image_type = ContainerImageType.LECTURE.toString(); + bind_mount_config = new ArrayList<>(); + } + + public ContainerMeta(ContainerMeta containerMeta) { + build_context_method = containerMeta.build_context_method; + build_context_url = containerMeta.build_context_url; + image_name = containerMeta.image_name; + run_options = containerMeta.run_options; + run_command = containerMeta.run_command; + image_repo = containerMeta.image_repo; + + for (ContainerBindMount bm : containerMeta.bind_mount_config) + bind_mount_config.add(new ContainerBindMount(bm.getSource(), bm.getTarget(), bm.getOptions())); + + } + + public int getBuildContextMethod() { + return build_context_method; + } + + public void setBuildContextMethod(int buildContextMethod) { + this.build_context_method = buildContextMethod; + } + + public String getBuildContextUrl() { + return build_context_url; + } + + public void setBuildContextUrl(String buildContextUrl) { + this.build_context_url = buildContextUrl; + } + + public String getRunOptions() { + return run_options; + } + + public void setRunOptions(String run_options) { + this.run_options = run_options; + } + + public void setRunCommand(String run_command) { + this.run_command = run_command; + } + + public String getRunCommand() { + return this.run_command; + } + + public String getImageName() { + return image_name; + } + + public void setImageName(String image_name) { + this.image_name = image_name; + } + + public List getBindMountConfig() { + return bind_mount_config; + } + + public void setBindMountConfig(List bindMountConfig) { + this.bind_mount_config = bindMountConfig; + } + + public String getImageRepo() { + return image_repo; + } + + public void setImageRepo(String from_image) { + this.image_repo = from_image; + } + + public ContainerImageType getImageType() { + if (image_type == null || image_type.length() == 0) + return ContainerImageType.LECTURE; + + return ContainerImageType.getByName(image_type); + } + + public void setImageType(ContainerImageType image_type) { + this.image_type = image_type.toString(); + } + + @Override public boolean equals(Object o) { + if (this == o) + return true; + if (o == null || getClass() != o.getClass()) + return false; + ContainerMeta that = (ContainerMeta) o; + return Objects.equals(build_context_url, that.build_context_url) && Objects.equals(image_name, + that.image_name) && Objects.equals(run_options, that.run_options) && Objects.equals( + run_command, that.run_command) && Objects.equals(bind_mount_config, that.bind_mount_config) + && Objects.equals(image_repo, that.image_repo) && Objects.equals(image_type, that.image_type); + } + + @Override public int hashCode() { + return Objects.hash(build_context_url, image_name, run_options, run_command, bind_mount_config, + image_repo, image_type); + } +} -- cgit v1.2.3-55-g7522