[DownloadFiles] Task for batch downloads

Supports using custom GPG pubkey for verification of downloaded files

1 files changed, 289 insertions, 0 deletions

diff --git a/src/main/java/org/openslx/taskmanager/tasks/DownloadFiles.java b/src/main/java/org/openslx/taskmanager/tasks/DownloadFiles.java
new file mode 100644
index 0000000..c274462
--- /dev/null
+++ b/src/main/java/org/openslx/taskmanager/tasks/DownloadFiles.java
@@ -0,0 +1,289 @@
+package org.openslx.taskmanager.tasks;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.net.URLConnection;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.attribute.PosixFilePermissions;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.FilenameUtils;
+import org.openslx.satserver.util.Exec;
+import org.openslx.satserver.util.ProxyHandler;
+import org.openslx.satserver.util.Util;
+import org.openslx.taskmanager.api.AbstractTask;
+
+import com.google.gson.annotations.Expose;
+
+public class DownloadFiles extends AbstractTask
+{
+
+	@Expose
+	private Task[] files = null;
+	
+	@Expose
+	private String baseDir;
+	
+	@Expose
+	private String gpgPubKey;
+
+	private Output statusObject = new Output();
+
+	private static final String[] ALLOWED_DIRS =
+	{ "/srv/openslx/www/boot/", "/tmp/" };
+
+	@Override
+	protected boolean initTask()
+	{
+		this.setStatusObject( statusObject );
+		if ( this.files == null || this.files.length == 0 ) {
+			statusObject.error = "No files given.";
+			return false;
+		}
+		if ( Util.isEmpty( this.baseDir ) ) {
+			statusObject.error = "No baseDir given.";
+			return false;
+		}
+		if ( this.baseDir.endsWith( "/" ) ) {
+			this.baseDir = this.baseDir.replaceAll( "/+$", "" );
+		}
+		for ( Task t : files ) {
+			String path = FilenameUtils.normalize( this.baseDir + "/" + t.fileName );
+			if ( path == null || !Util.startsWith( path, ALLOWED_DIRS )
+					|| path.endsWith( "/" ) || !Util.startsWith( path, this.baseDir ) ) {
+				statusObject.error = "File '" + t.fileName + "' not in allowed directory";
+				return false;
+			}
+		}
+		return true;
+	}
+
+	@Override
+	protected boolean execute()
+	{
+		// Keyring
+		final File gpgDir;
+		if ( this.gpgPubKey != null ) {
+			Path pubkeyPath;
+			try {
+				pubkeyPath = Files.createTempDirectory( "bwlp-",
+						PosixFilePermissions.asFileAttribute( PosixFilePermissions.fromString( "rwx------" ) ) );
+			} catch ( Exception e ) {
+				statusObject.error = "Cannot create temp dir for gpg pubkey";
+				return false;
+			}
+			gpgDir = pubkeyPath.toFile();
+			File pubkey = new File( gpgDir, "import-file.asc" );
+			try {
+				FileUtils.write( pubkey, this.gpgPubKey, StandardCharsets.UTF_8 );
+			} catch ( IOException e ) {
+				statusObject.error = "Cannot write gpg pubkey to tempfile";
+				return false;
+			}
+			int ret = Exec.sync( 3, "gpg", "--batch", "--homedir", gpgDir.getAbsolutePath(), "--import", pubkey.getAbsolutePath() );
+			if ( ret != 0 ) {
+				statusObject.error = "gpg key import failed with exit code " + ret;
+				return false;
+			}
+		} else {
+			gpgDir = null;
+		}
+		// Create temp dir
+		Path td = null;
+		Exception ex = null;
+		for(int x = 0; x < 10; ++x) {
+			String rnd = Double.toString( Math.random() );
+			try {
+				td = Files.createDirectories( Paths.get( this.baseDir + "-" + rnd + ".tmp" ) );
+				break;
+			} catch (Exception e) {
+				ex = e;
+			}
+		}
+		if ( td == null ) {
+			statusObject.error = "Cannot create temporary directory: " + ex;
+			return false;
+		}
+		final Path tmpDir = td;
+		// Handle proxy settings before opening connection for downloading.
+		ProxyHandler.configProxy();
+		ExecutorService tp = Executors.newFixedThreadPool( files.length > 3 ? 3 : files.length );
+		final AtomicBoolean retval = new AtomicBoolean( true );
+		for ( final Task t : this.files ) {
+			final FileStatus status = new FileStatus();
+			status.id = t.id;
+			statusObject.files.add( status );
+			tp.submit( new Runnable() {
+				public void run()
+				{
+					URLConnection connection = null;
+					InputStream in = null;
+					FileOutputStream fout = null;
+					try {
+						File tmpFile = new File( tmpDir.toFile(), t.fileName );
+						connection = new URL( t.url ).openConnection();
+						in = connection.getInputStream();
+						fout = new FileOutputStream( tmpFile );
+						status.size = connection.getContentLengthLong();
+						if ( status.size <= 0 ) { // If size is unknown, fake progress...
+							status.progress = 10;
+						}
+						final byte data[] = new byte[ 90000 ];
+						int count;
+						while ( ( count = in.read( data, 0, data.length ) ) != -1 ) {
+							fout.write( data, 0, count );
+							status.complete += count;
+							if ( status.size > 0 )
+								status.progress = (int) ( 100l * status.complete / status.size );
+							else if ( status.progress < 95 && System.currentTimeMillis() % 50 == 0 )
+								status.progress++;
+						}
+						// If we have a gpg sig, validate
+						if ( !Util.isEmpty( t.gpg ) ) {
+							if ( !checkSig( t, status, tmpFile, gpgDir ) ) {
+								retval.set( false );
+								return;
+							}
+							status.progress = 100;
+						}
+					} catch ( Exception e ) {
+						status.error = "Download error: " + e.toString();
+						retval.set( false );
+					} finally {
+						Util.multiClose( fout );
+						flushStream( in );
+						if ( connection instanceof HttpURLConnection ) {
+							InputStream es = ( (HttpURLConnection)connection ).getErrorStream();
+							flushStream( es );
+							Util.multiClose( es );
+						}
+					}
+				}
+			} );
+		}
+		tp.shutdown();
+		try {
+			tp.awaitTermination( 5, TimeUnit.MINUTES );
+		} catch ( InterruptedException e ) {
+			Thread.currentThread().interrupt();
+		}
+		try {
+			FileUtils.forceDelete( gpgDir );
+		} catch ( Exception e1 ) {
+		}
+		if ( retval.get() ) {
+			// Move dir to proper location
+			File dest = new File( this.baseDir );
+			try {
+				FileUtils.forceDelete( dest );
+			} catch ( FileNotFoundException e ) {
+				// Ignore
+			} catch ( Exception e ) {
+				statusObject.error = "Cannot delete target dir " + this.baseDir + ": " + e.toString();
+				retval.set( false );
+			}
+			if ( retval.get() && !tmpDir.toFile().renameTo( dest ) ) {
+				statusObject.error = "Cannot rename temporary download directory " + tmpDir.toString() + " to " + dest.toString();
+				retval.set( false );
+			}
+		}
+		if ( !retval.get() ) {
+			try {
+				FileUtils.forceDelete( tmpDir.toFile() );
+			} catch ( Exception e ) {
+			}
+		}
+		return retval.get();
+	}
+
+	private boolean checkSig( Task t, FileStatus status, File fileToCheck, File gpgDir )
+	{
+		File gpgTempFile = null;
+		try {
+			try {
+				gpgTempFile = File.createTempFile( "bwlp-", ".gpg", null );
+				Util.writeStringToFile( gpgTempFile, t.gpg );
+			} catch ( Exception e ) {
+				status.error = "Could not create temporary file for gpg signature";
+				return false;
+			}
+			ArrayList<String> args = new ArrayList<>();
+			args.add( "gpg" );
+			if ( gpgDir != null ) {
+				args.add( "--homedir" );
+				args.add( gpgDir.getAbsolutePath() );
+			}
+			args.add( "--batch" );
+			args.add( "--verify" );
+			args.add( gpgTempFile.getAbsolutePath() );
+			args.add( fileToCheck.getAbsolutePath() );
+			if ( 0 != Exec.sync( 10, args.toArray( new String[args.size()] ) ) ) {
+				status.error = "GPG signature of downloaded file not valid!\n\n" + t.gpg;
+				return false;
+			}
+			return true;
+		} finally {
+			if ( gpgTempFile != null && gpgTempFile.exists() )
+				gpgTempFile.delete();
+		}
+	}
+	
+	private void flushStream( InputStream is )
+	{
+		if ( is == null )
+			return;
+		byte buffer[] = new byte[ 10000 ];
+		try {
+			while ( is.read( buffer ) != -1 );
+		} catch ( IOException e ) {
+		}
+	}
+
+	/**
+	 * Output - contains additional status data of this task
+	 */
+	@SuppressWarnings( "unused" )
+	private static class Output
+	{
+		protected String error = null;
+		protected List<FileStatus> files = new CopyOnWriteArrayList<>();
+	}
+	
+	@SuppressWarnings( "unused" )
+	private static class FileStatus
+	{
+		protected String error = null;
+		protected long size = -1;
+		protected long complete = 0;
+		protected int progress = 0;
+		protected String id;
+	}
+
+	private static class Task
+	{
+		@Expose
+		public String id;
+		@Expose
+		public String url;
+		@Expose
+		public String fileName;
+		@Expose
+		public String gpg;
+	}
+
+}