From a143ba9f0a98e530178b494cf7542a71e7a7f8fc Mon Sep 17 00:00:00 2001 From: Simon Rettberg Date: Fri, 22 Feb 2019 17:28:41 +0100 Subject: [*] Make ServerSideCopy configurable Options are OFF, ON, AUTO, USER AUTO enables it as soon as upload speed falls below 10MiB/s USER enables a checkbox in the client to toggle it for each individual upload --- .../openslx/dozmod/gui/activity/UploadPanel.java | 4 +- .../java/org/openslx/bwlp/sat/RuntimeConfig.java | 5 ++ .../bwlp/sat/fileserv/IncomingDataTransfer.java | 62 +++++++++++++++++++--- 3 files changed, 64 insertions(+), 7 deletions(-) diff --git a/dozentenmodul/src/main/java/org/openslx/dozmod/gui/activity/UploadPanel.java b/dozentenmodul/src/main/java/org/openslx/dozmod/gui/activity/UploadPanel.java index ea339651..df396b55 100644 --- a/dozentenmodul/src/main/java/org/openslx/dozmod/gui/activity/UploadPanel.java +++ b/dozentenmodul/src/main/java/org/openslx/dozmod/gui/activity/UploadPanel.java @@ -8,6 +8,7 @@ import javax.swing.JCheckBox; import org.apache.log4j.Logger; import org.apache.thrift.TException; +import org.openslx.bwlp.thrift.iface.SscMode; import org.openslx.bwlp.thrift.iface.TAuthorizationException; import org.openslx.bwlp.thrift.iface.TInvalidTokenException; import org.openslx.bwlp.thrift.iface.UploadOptions; @@ -34,7 +35,8 @@ public class UploadPanel extends TransferPanel { public UploadPanel(UploadWizardState state) { super(state.upload.getUploadTask(), state.name, state.diskFile.getName()); - if (!Session.hasFeature(Feature.SERVER_SIDE_COPY)) { + if (!Session.hasFeature(Feature.SERVER_SIDE_COPY) + || Session.getSatelliteConfig().serverSideCopy != SscMode.USER) { chkServerSideCopy = null; } else { chkServerSideCopy = new JCheckBox("ServerSide Copy"); diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/RuntimeConfig.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/RuntimeConfig.java index df0349fa..dbdf29cc 100644 --- a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/RuntimeConfig.java +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/RuntimeConfig.java @@ -8,6 +8,7 @@ import org.openslx.bwlp.sat.util.Constants; import org.openslx.bwlp.thrift.iface.ImagePermissions; import org.openslx.bwlp.thrift.iface.LecturePermissions; import org.openslx.bwlp.thrift.iface.SatelliteConfig; +import org.openslx.bwlp.thrift.iface.SscMode; import org.openslx.util.GenericDataCache; public class RuntimeConfig { @@ -56,6 +57,10 @@ public class RuntimeConfig { if (satConfig.maxLocationsPerLecture == -1) { satConfig.setMaxLocationsPerLecture(4); } + if (satConfig.serverSideCopy == null) { + satConfig.serverSideCopy = SscMode.OFF; + } + // Update if we sanitized or added anything if (!satConfig.equals(readConfig)) { try { DbConfiguration.setSatelliteConfig(satConfig); diff --git a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/fileserv/IncomingDataTransfer.java b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/fileserv/IncomingDataTransfer.java index d5dd5a14..aa3047af 100644 --- a/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/fileserv/IncomingDataTransfer.java +++ b/dozentenmodulserver/src/main/java/org/openslx/bwlp/sat/fileserv/IncomingDataTransfer.java @@ -9,10 +9,12 @@ import java.util.List; import java.util.UUID; import java.util.concurrent.ExecutorService; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; import javax.net.ssl.SSLContext; import org.apache.log4j.Logger; +import org.openslx.bwlp.sat.RuntimeConfig; import org.openslx.bwlp.sat.database.mappers.DbImage; import org.openslx.bwlp.sat.database.mappers.DbImageBlock; import org.openslx.bwlp.sat.database.mappers.DbLog; @@ -23,6 +25,7 @@ import org.openslx.bwlp.sat.util.Formatter; import org.openslx.bwlp.thrift.iface.ImageDetailsRead; import org.openslx.bwlp.thrift.iface.ImagePublishData; import org.openslx.bwlp.thrift.iface.ImageVersionWrite; +import org.openslx.bwlp.thrift.iface.SscMode; import org.openslx.bwlp.thrift.iface.TNotFoundException; import org.openslx.bwlp.thrift.iface.TransferInformation; import org.openslx.bwlp.thrift.iface.TransferState; @@ -88,7 +91,7 @@ public class IncomingDataTransfer extends IncomingTransferBase { this.image = image; this.machineDescription = machineDescription; this.masterTransferInfo = null; - initRepairUpload(); + initCommonUpload(); } public IncomingDataTransfer(ImagePublishData publishData, File tmpFile, TransferInformation transferInfo, @@ -113,10 +116,17 @@ public class IncomingDataTransfer extends IncomingTransferBase { this.machineDescription = ThriftUtil.unwrapByteBuffer(transferInfo.machineDescription); this.masterTransferInfo = transferInfo; this.versionSettings = new ImageVersionWrite(false); - initRepairUpload(); + initCommonUpload(); } - private void initRepairUpload() { + private void initCommonUpload() { + SscMode sscMode = RuntimeConfig.get().serverSideCopy; + if (sscMode == SscMode.OFF) { + super.enableServerSideCopying(false); + } else if (sscMode == SscMode.ON) { + super.enableServerSideCopying(true); + } + // Handle repair upload... if (!isRepairUpload()) return; if (getTmpFileName().exists() && getTmpFileName().length() > 0) { @@ -372,9 +382,46 @@ public class IncomingDataTransfer extends IncomingTransferBase { } } } + + // Measure speed for automatic server-side copy + private final AtomicInteger speedCounter = new AtomicInteger(); + private long speedTimestamp = 0; + private static final long SSC_ENABLE_THRES = 10l * 1024 * 1024; + private static final long SSC_DISABLE_THRES = 20l * 1024 * 1024; @Override protected boolean chunkReceived(FileChunk chunk, byte[] data) { + SscMode sscMode = RuntimeConfig.get().serverSideCopy; + if (sscMode == SscMode.AUTO) { + // Automatic SSC setting + long diff = 0; + long bytes; + synchronized (speedCounter) { + bytes = speedCounter.addAndGet(chunk.range.getLength()); + if (bytes >= FileChunk.CHUNK_SIZE * 3) { + diff = System.currentTimeMillis() - speedTimestamp; + speedTimestamp = System.currentTimeMillis(); + } + } + if (diff != 0 && diff < 100000000) { + // Time to evaluate the situation + long speed = bytes / (diff / 1000); + if (speed < SSC_ENABLE_THRES) { + super.enableServerSideCopying(true); + } else if (speed > SSC_DISABLE_THRES) { + super.enableServerSideCopying(false); + } + } + + } else { + speedTimestamp = 0; + if (sscMode == SscMode.OFF) { + super.enableServerSideCopying(false); + } else if (sscMode == SscMode.ON) { + super.enableServerSideCopying(true); + } + } + // Hashing if (getHashChecker() == null) return false; try { @@ -394,9 +441,12 @@ public class IncomingDataTransfer extends IncomingTransferBase { * Alter options of this upload. Returns new effective options. */ public UploadOptions setOptions(UploadOptions options) { - if (options != null) { - if (options.isSetServerSideCopying()) { - super.enableServerSideCopying(options.serverSideCopying); + if (RuntimeConfig.get().serverSideCopy == SscMode.USER) { + // User can fiddle around + if (options != null) { + if (options.isSetServerSideCopying()) { + super.enableServerSideCopying(options.serverSideCopying); + } } } return new UploadOptions(super.isServerSideCopyingEnabled()); -- cgit v1.2.3-55-g7522