From 1c2fd750b71aa939b723624fb8bc654e19e9abb0 Mon Sep 17 00:00:00 2001 From: ralph isenmann Date: Tue, 7 Sep 2021 08:44:01 +0200 Subject: [server] add endpoint to dmsd webserver for image metadata new endpoint /image/container//metadata to request specific information about an existing image (virtId==docker and containerImageType==data) --- .../openslx/bwlp/sat/database/mappers/DbImage.java | 67 +++++++++++++++------- .../java/org/openslx/bwlp/sat/web/WebServer.java | 16 ++++++ 2 files changed, 61 insertions(+), 22 deletions(-) (limited to 'dozentenmodulserver/src/main/java') diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbImage.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbImage.java index bac3ad6c..62041952 100644 --- a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbImage.java +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbImage.java @@ -1,14 +1,6 @@ package org.openslx.bwlp.sat.database.mappers; -import java.io.*; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.UUID; - +import com.google.gson.JsonObject; import org.apache.log4j.Logger; import org.openslx.bwlp.sat.RuntimeConfig; import org.openslx.bwlp.sat.database.Database; @@ -20,27 +12,20 @@ import org.openslx.bwlp.sat.database.models.LocalImageVersion; import org.openslx.bwlp.sat.mail.MailGenerator; import org.openslx.bwlp.sat.permissions.User; import org.openslx.bwlp.sat.util.FileSystem; -import org.openslx.bwlp.thrift.iface.ImageBaseWrite; -import org.openslx.bwlp.thrift.iface.ImageDetailsRead; -import org.openslx.bwlp.thrift.iface.ImagePermissions; -import org.openslx.bwlp.thrift.iface.ImagePublishData; -import org.openslx.bwlp.thrift.iface.ImageSummaryRead; -import org.openslx.bwlp.thrift.iface.ImageVersionDetails; -import org.openslx.bwlp.thrift.iface.ImageVersionWrite; -import org.openslx.bwlp.thrift.iface.Role; -import org.openslx.bwlp.thrift.iface.ShareMode; -import org.openslx.bwlp.thrift.iface.TNotFoundException; -import org.openslx.bwlp.thrift.iface.UserInfo; +import org.openslx.bwlp.thrift.iface.*; import org.openslx.filetransfer.util.ChunkList; import org.openslx.filetransfer.util.FileChunk; import org.openslx.util.QuickTimer; import org.openslx.util.QuickTimer.Task; import org.openslx.util.Util; - -// master-sync-shared import org.openslx.virtualization.configuration.container.ContainerDefinition; import org.openslx.virtualization.configuration.container.ContainerMeta; +import java.io.File; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.*; + public class DbImage { private static final Logger LOGGER = Logger.getLogger(DbImage.class); @@ -1162,6 +1147,44 @@ public class DbImage { } } + public static String getContainerImageMetadata(String imagebaseid) throws SQLException { + + try (MysqlConnection connection = Database.getConnection()) { + MysqlStatement stmt = connection.prepareStatement( + "SELECT ib.imagebaseid, ib.displayname, iv.filepath, iv.filesize, iv.virtualizerconfig" + + " FROM imagebase AS ib" + + " JOIN imageversion AS iv ON (ib.latestversionid=iv.imageversionid AND ib.virtid = 'docker' AND ib.latestversionid IS NOT NULL)" + + " WHERE ib.imagebaseid = :imagebaseid"); + stmt.setString("imagebaseid", imagebaseid); + ResultSet rs = stmt.executeQuery(); + + JsonObject resultJson = new JsonObject(); + while (rs.next()) { + ContainerDefinition condev = ContainerDefinition.fromByteArray( + rs.getBytes("iv.virtualizerconfig")); + // currently only data images are returned + if (condev.getContainerMeta().getImageType() != ContainerMeta.ContainerImageType.DATA) + break; + + resultJson.addProperty("displayname", rs.getString("ib.displayname")); + resultJson.addProperty("imagepath", rs.getString("iv.filepath")); + resultJson.addProperty("filesize", rs.getString("iv.filesize")); + + resultJson.addProperty("image_recipe", condev.getContainerRecipe()); + resultJson.addProperty("image_repo", condev.getContainerMeta().getImageRepo()); + resultJson.addProperty("build_context_method", + condev.getContainerMeta().getBuildContextMethod()); + resultJson.addProperty("build_context_url", condev.getContainerMeta().getBuildContextUrl()); + break; + } + + return resultJson.toString(); + } catch (Exception e) { + LOGGER.error("Query failed in DbImage.getContainerImages()", e); + throw e; + } + } + static class ContainerImages { public final String owner_firstname; public final String owner_lastname; diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebServer.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebServer.java index a3d9d293..cc9f8b72 100644 --- a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebServer.java +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebServer.java @@ -99,6 +99,12 @@ public class WebServer extends NanoHTTPD { if (uri.startsWith("/bwlp/container/clusterimages")) { return serverContainerImages(); } + if (uri.startsWith("/image/container/")) { + if (parts.length < 4) + return badRequest("Bad Request"); + if (parts[3].equals("metadata")) + return serveContainerImageMetaData(parts[2]); + } if (uri.startsWith("/status/fileserver")) { return serveStatus(); @@ -310,4 +316,14 @@ public class WebServer extends NanoHTTPD { return internalServerError(); } } + + private Response serveContainerImageMetaData(String imageBaseId) { + try { + return new Response(Response.Status.OK, "application/json; charset=utf-8", + DbImage.getContainerImageMetadata(imageBaseId)); + } catch (SQLException e) { + LOGGER.error("error -- could not server container image", e); + return internalServerError(); + } + } } -- cgit v1.2.3-55-g7522 From 467cb8248dc3c0d2551345346f564a4b04d9c3be Mon Sep 17 00:00:00 2001 From: ralph isenmann Date: Tue, 7 Sep 2021 08:44:01 +0200 Subject: [server] add endpoint to dmsd webserver for image metadata new endpoint /image/container//metadata to request specific information about an existing image (virtId==docker and containerImageType==data) --- .../openslx/bwlp/sat/database/mappers/DbImage.java | 67 +++++++++++++++------- .../java/org/openslx/bwlp/sat/web/WebServer.java | 16 ++++++ 2 files changed, 61 insertions(+), 22 deletions(-) (limited to 'dozentenmodulserver/src/main/java') diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbImage.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbImage.java index bac3ad6c..62041952 100644 --- a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbImage.java +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbImage.java @@ -1,14 +1,6 @@ package org.openslx.bwlp.sat.database.mappers; -import java.io.*; -import java.sql.ResultSet; -import java.sql.SQLException; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.UUID; - +import com.google.gson.JsonObject; import org.apache.log4j.Logger; import org.openslx.bwlp.sat.RuntimeConfig; import org.openslx.bwlp.sat.database.Database; @@ -20,27 +12,20 @@ import org.openslx.bwlp.sat.database.models.LocalImageVersion; import org.openslx.bwlp.sat.mail.MailGenerator; import org.openslx.bwlp.sat.permissions.User; import org.openslx.bwlp.sat.util.FileSystem; -import org.openslx.bwlp.thrift.iface.ImageBaseWrite; -import org.openslx.bwlp.thrift.iface.ImageDetailsRead; -import org.openslx.bwlp.thrift.iface.ImagePermissions; -import org.openslx.bwlp.thrift.iface.ImagePublishData; -import org.openslx.bwlp.thrift.iface.ImageSummaryRead; -import org.openslx.bwlp.thrift.iface.ImageVersionDetails; -import org.openslx.bwlp.thrift.iface.ImageVersionWrite; -import org.openslx.bwlp.thrift.iface.Role; -import org.openslx.bwlp.thrift.iface.ShareMode; -import org.openslx.bwlp.thrift.iface.TNotFoundException; -import org.openslx.bwlp.thrift.iface.UserInfo; +import org.openslx.bwlp.thrift.iface.*; import org.openslx.filetransfer.util.ChunkList; import org.openslx.filetransfer.util.FileChunk; import org.openslx.util.QuickTimer; import org.openslx.util.QuickTimer.Task; import org.openslx.util.Util; - -// master-sync-shared import org.openslx.virtualization.configuration.container.ContainerDefinition; import org.openslx.virtualization.configuration.container.ContainerMeta; +import java.io.File; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.*; + public class DbImage { private static final Logger LOGGER = Logger.getLogger(DbImage.class); @@ -1162,6 +1147,44 @@ public class DbImage { } } + public static String getContainerImageMetadata(String imagebaseid) throws SQLException { + + try (MysqlConnection connection = Database.getConnection()) { + MysqlStatement stmt = connection.prepareStatement( + "SELECT ib.imagebaseid, ib.displayname, iv.filepath, iv.filesize, iv.virtualizerconfig" + + " FROM imagebase AS ib" + + " JOIN imageversion AS iv ON (ib.latestversionid=iv.imageversionid AND ib.virtid = 'docker' AND ib.latestversionid IS NOT NULL)" + + " WHERE ib.imagebaseid = :imagebaseid"); + stmt.setString("imagebaseid", imagebaseid); + ResultSet rs = stmt.executeQuery(); + + JsonObject resultJson = new JsonObject(); + while (rs.next()) { + ContainerDefinition condev = ContainerDefinition.fromByteArray( + rs.getBytes("iv.virtualizerconfig")); + // currently only data images are returned + if (condev.getContainerMeta().getImageType() != ContainerMeta.ContainerImageType.DATA) + break; + + resultJson.addProperty("displayname", rs.getString("ib.displayname")); + resultJson.addProperty("imagepath", rs.getString("iv.filepath")); + resultJson.addProperty("filesize", rs.getString("iv.filesize")); + + resultJson.addProperty("image_recipe", condev.getContainerRecipe()); + resultJson.addProperty("image_repo", condev.getContainerMeta().getImageRepo()); + resultJson.addProperty("build_context_method", + condev.getContainerMeta().getBuildContextMethod()); + resultJson.addProperty("build_context_url", condev.getContainerMeta().getBuildContextUrl()); + break; + } + + return resultJson.toString(); + } catch (Exception e) { + LOGGER.error("Query failed in DbImage.getContainerImages()", e); + throw e; + } + } + static class ContainerImages { public final String owner_firstname; public final String owner_lastname; diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebServer.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebServer.java index a3d9d293..cc9f8b72 100644 --- a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebServer.java +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebServer.java @@ -99,6 +99,12 @@ public class WebServer extends NanoHTTPD { if (uri.startsWith("/bwlp/container/clusterimages")) { return serverContainerImages(); } + if (uri.startsWith("/image/container/")) { + if (parts.length < 4) + return badRequest("Bad Request"); + if (parts[3].equals("metadata")) + return serveContainerImageMetaData(parts[2]); + } if (uri.startsWith("/status/fileserver")) { return serveStatus(); @@ -310,4 +316,14 @@ public class WebServer extends NanoHTTPD { return internalServerError(); } } + + private Response serveContainerImageMetaData(String imageBaseId) { + try { + return new Response(Response.Status.OK, "application/json; charset=utf-8", + DbImage.getContainerImageMetadata(imageBaseId)); + } catch (SQLException e) { + LOGGER.error("error -- could not server container image", e); + return internalServerError(); + } + } } -- cgit v1.2.3-55-g7522 From da4e3ed2dab82bf638e7fdf06cefd6855ae8f9c7 Mon Sep 17 00:00:00 2001 From: ralph isenmann Date: Wed, 10 Nov 2021 14:00:16 +0100 Subject: [server] update webserver and dbimage; refactoring --- .../openslx/bwlp/sat/database/mappers/DbImage.java | 2 +- .../java/org/openslx/bwlp/sat/web/WebServer.java | 50 +++++++++------------- 2 files changed, 22 insertions(+), 30 deletions(-) (limited to 'dozentenmodulserver/src/main/java') diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbImage.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbImage.java index 62041952..9c39078c 100644 --- a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbImage.java +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/database/mappers/DbImage.java @@ -1173,7 +1173,7 @@ public class DbImage { resultJson.addProperty("image_recipe", condev.getContainerRecipe()); resultJson.addProperty("image_repo", condev.getContainerMeta().getImageRepo()); resultJson.addProperty("build_context_method", - condev.getContainerMeta().getBuildContextMethod()); + condev.getContainerMeta().getContainerImageContext()); resultJson.addProperty("build_context_url", condev.getContainerMeta().getBuildContextUrl()); break; } diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebServer.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebServer.java index cc9f8b72..cb7b5bf9 100644 --- a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebServer.java +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/web/WebServer.java @@ -41,9 +41,9 @@ import fi.iki.elonen.NanoHTTPD; public class WebServer extends NanoHTTPD { private static final Logger LOGGER = Logger.getLogger(WebServer.class); - - private static final ThreadPoolExecutor tpe = - new GrowingThreadPoolExecutor(1, 8, 1, TimeUnit.MINUTES, new LinkedBlockingQueue(16)); + + private static final ThreadPoolExecutor tpe = new GrowingThreadPoolExecutor(1, 8, 1, TimeUnit.MINUTES, + new LinkedBlockingQueue(16)); private static final Serializer serializer = new Persister(); @@ -93,18 +93,14 @@ public class WebServer extends NanoHTTPD { return serveMetaData(parts[2]); if (parts[3].equals("netrules")) return serveLectureNetRules(parts[2]); + if (parts[3].equals("imagemeta")) + return serveContainerImageMetaData(parts[2]); } return notFound(); } if (uri.startsWith("/bwlp/container/clusterimages")) { return serverContainerImages(); } - if (uri.startsWith("/image/container/")) { - if (parts.length < 4) - return badRequest("Bad Request"); - if (parts[3].equals("metadata")) - return serveContainerImageMetaData(parts[2]); - } if (uri.startsWith("/status/fileserver")) { return serveStatus(); @@ -126,16 +122,14 @@ public class WebServer extends NanoHTTPD { return new NanoHTTPD.Response(NanoHTTPD.Response.Status.OK, "application/json; charset=utf-8", Json.serialize(FileServer.instance().getStatus())); } - - private static void tarPutFile(TarOutputStream output, String fileName, String data) throws IOException - { + + private static void tarPutFile(TarOutputStream output, String fileName, String data) throws IOException { if (data == null) return; tarPutFile(output, fileName, data.getBytes(StandardCharsets.UTF_8)); } - - private static void tarPutFile(TarOutputStream output, String fileName, byte[] data) throws IOException - { + + private static void tarPutFile(TarOutputStream output, String fileName, byte[] data) throws IOException { if (data == null) return; output.putNextEntry(new TarEntry(TarHeader.createHeader(fileName, data.length, Util.unixTime(), false, 0644))); @@ -172,8 +166,7 @@ public class WebServer extends NanoHTTPD { if (ld.runScript != null) { int cnt = 0; for (RunScript rs : ld.runScript) { - tarPutFile(output, String.format("adminrun/%04d-%d-%d.%s", - cnt++, rs.visibility, + tarPutFile(output, String.format("adminrun/%04d-%d-%d.%s", cnt++, rs.visibility, rs.passCreds ? 1 : 0, rs.extension), rs.content); } } @@ -191,10 +184,9 @@ public class WebServer extends NanoHTTPD { LOGGER.warn("Server overloaded; rejecting VM Metadata request", e2); return internalServerError(); } - return new NanoHTTPD.Response(NanoHTTPD.Response.Status.OK, "application/gzip", - sink); + return new NanoHTTPD.Response(NanoHTTPD.Response.Status.OK, "application/gzip", sink); } - + private Response serveLectureNetRules(String lectureId) { List list = new ArrayList<>(); boolean defaultAllowed; @@ -223,13 +215,12 @@ public class WebServer extends NanoHTTPD { sb.append("IN * 0 REJECT\n"); sb.append("OUT * 0 REJECT\n"); } - return new NanoHTTPD.Response(NanoHTTPD.Response.Status.OK, "text/plain; charset=utf-8", - sb.toString()); + return new NanoHTTPD.Response(NanoHTTPD.Response.Status.OK, "text/plain; charset=utf-8", sb.toString()); } private String serializeNetShares(List list) { // openslx.exe expects shares in the following format - // + // // letter is either a drive letter for Windows VMs, // or a mount point for Linux VMs. StringBuilder sb = new StringBuilder(); @@ -276,13 +267,13 @@ public class WebServer extends NanoHTTPD { /** * Helper for returning "Internal Server Error" Status - * @param body Message + * + * @param body Message */ public static Response internalServerError(String body) { - return new NanoHTTPD.Response(NanoHTTPD.Response.Status.INTERNAL_ERROR, "text/plain", - body); + return new NanoHTTPD.Response(NanoHTTPD.Response.Status.INTERNAL_ERROR, "text/plain", body); } - + public static Response internalServerError() { return internalServerError("Internal Server Error"); } @@ -305,9 +296,10 @@ public class WebServer extends NanoHTTPD { } /** - * create a json response with information about existing container images in bwlehrpool + * create a json response with information about existing container images in + * bwlehrpool */ - private Response serverContainerImages () { + private Response serverContainerImages() { try { return new Response(Response.Status.OK, "application/json; charset=utf-8", Json.serialize(DbImage.getContainerImageCluster())); -- cgit v1.2.3-55-g7522